1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
|
{
lib,
buildPythonPackage,
fetchFromGitHub,
# build-system
setuptools,
# dependencies
absl-py,
etils,
jsonpath-rw,
networkx,
pandas,
pandas-stubs,
python-dateutil,
rdflib,
requests,
scipy,
tqdm,
# tests
apache-beam,
gitpython,
librosa,
pillow,
pytestCheckHook,
pyyaml,
writableTmpDirAsHomeHook,
}:
buildPythonPackage rec {
pname = "mlcroissant";
version = "1.0.22";
pyproject = true;
src = fetchFromGitHub {
owner = "mlcommons";
repo = "croissant";
tag = "v${version}";
hash = "sha256-uJOxKNrK3eN2wyPFEQr2J4+vZeSK1KPyFDag2jcyWZw=";
};
sourceRoot = "${src.name}/python/mlcroissant";
build-system = [
setuptools
];
dependencies = [
absl-py
etils
jsonpath-rw
networkx
pandas
pandas-stubs
python-dateutil
rdflib
requests
scipy
tqdm
]
++ etils.optional-dependencies.epath;
pythonImportsCheck = [ "mlcroissant" ];
nativeCheckInputs = [
apache-beam
gitpython
librosa
pillow
pytestCheckHook
pyyaml
writableTmpDirAsHomeHook
];
disabledTests = [
# Requires internet access
"test_hermetic_loading_1_1"
"test_load_from_huggingface"
"test_nonhermetic_loading"
"test_nonhermetic_loading_1_0"
# AssertionError: assert {'records/aud...t32), 22050)'} == {'records/aud...t32), 22050)'}
"test_hermetic_loading"
# AttributeError: 'MaybeReshuffle' object has no attribute 'side_inputs'
"test_beam_hermetic_loading"
];
meta = {
description = "High-level format for machine learning datasets that brings together four rich layers";
homepage = "https://github.com/mlcommons/croissant";
changelog = "https://github.com/mlcommons/croissant/releases/tag/${src.tag}";
license = lib.licenses.asl20;
maintainers = with lib.maintainers; [ GaetanLepage ];
platforms = lib.platforms.all;
mainProgram = "mlcroissant";
};
}
|
