Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/datasets/features/features.py
Original file line number Diff line number Diff line change
Expand Up @@ -2325,8 +2325,8 @@ def reorder_fields_as(self, other: "Features") -> "Features":

>>> from datasets import Features, List, Value
>>> # let's say we have two features with a different order of nested fields (for a and b for example)
>>> f1 = Features({"root": {"a": Value("string"), "b": Value("string")}})
>>> f2 = Features({"root": {"b": Value("string"), "a": Value("string")}})
>>> f1 = Features({"root": List({"a": Value("string"), "b": Value("string")})})
>>> f2 = Features({"root": List({"b": Value("string"), "a": Value("string")})})
>>> assert f1.type != f2.type
>>> # re-ordering keeps the base structure (here List is defined at the root level), but makes the fields order match
>>> f1.reorder_fields_as(f2)
Expand Down Expand Up @@ -2378,8 +2378,8 @@ def flatten(self, max_depth=16) -> "Features":
>>> from datasets import load_dataset
>>> ds = load_dataset("rajpurkar/squad", split="train")
>>> ds.features.flatten()
{'answers.answer_start': List(Value('int32'), id=None),
'answers.text': List(Value('string'), id=None),
{'answers.answer_start': List(Value('int32')),
'answers.text': List(Value('string')),
'context': Value('string'),
'id': Value('string'),
'question': Value('string'),
Expand Down
3 changes: 2 additions & 1 deletion src/datasets/features/image.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,11 @@ class Image:
>>> from datasets import load_dataset, Image
>>> ds = load_dataset("AI-Lab-Makerere/beans", split="train")
>>> ds.features["image"]
Image(decode=True, id=None)
Image(mode=None, decode=True)
>>> ds[0]["image"]
<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=500x500 at 0x15E52E7F0>
>>> ds = ds.cast_column('image', Image(decode=False))
>>> ds[0]["image"]
{'bytes': None,
'path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/healthy/healthy_train.85.jpg'}
```
Expand Down
2 changes: 1 addition & 1 deletion src/datasets/features/nifti.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ class Nifti:
>>> from datasets import Dataset, Nifti
>>> ds = Dataset.from_dict({"nifti": ["path/to/file.nii.gz"]}).cast_column("nifti", Nifti())
>>> ds.features["nifti"]
Nifti(decode=True, id=None)
Nifti(decode=True)
>>> ds[0]["nifti"]
<nibabel.nifti1.Nifti1Image object at 0x7f8a1c2d8f40>
>>> ds = ds.cast_column("nifti", Nifti(decode=False))
Expand Down
2 changes: 1 addition & 1 deletion src/datasets/features/pdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class Pdf:
>>> from datasets import Dataset, Pdf
>>> ds = Dataset.from_dict({"pdf": ["path/to/pdf/file.pdf"]}).cast_column("pdf", Pdf())
>>> ds.features["pdf"]
Pdf(decode=True, id=None)
Pdf(decode=True)
>>> ds[0]["pdf"]
<pdfplumber.pdf.PDF object at 0x7f8a1c2d8f40>
>>> ds = ds.cast_column("pdf", Pdf(decode=False))
Expand Down
4 changes: 2 additions & 2 deletions src/datasets/features/video.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class Video:
>>> from datasets import Dataset, Video
>>> ds = Dataset.from_dict({"video":["path/to/Screen Recording.mov"]}).cast_column("video", Video())
>>> ds.features["video"]
Video(decode=True, id=None)
Video(decode=True, stream_index=None, dimension_order='NCHW', num_ffmpeg_threads=1, device='cpu', seek_mode='exact')
>>> ds[0]["video"]
<torchcodec.decoders._video_decoder.VideoDecoder object at 0x14a61e080>
>>> video = ds[0]["video"]
Expand All @@ -81,7 +81,7 @@ class Video:
0.4333], dtype=torch.float64)
duration_seconds: tensor([0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167,
0.0167], dtype=torch.float64)
>>> ds.cast_column('video', Video(decode=False))[0]["video]
>>> ds.cast_column('video', Video(decode=False))[0]["video"]
{'bytes': None,
'path': 'path/to/Screen Recording.mov'}
```
Expand Down
6 changes: 3 additions & 3 deletions src/datasets/iterable_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -4264,15 +4264,15 @@ def cast_column(self, column: str, feature: FeatureType) -> "IterableDataset":
>>> from datasets import load_dataset, Audio
>>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train", streaming=True)
>>> ds.features
{'audio': Audio(sampling_rate=8000, mono=True, decode=True, id=None),
{'audio': Audio(sampling_rate=8000, decode=True, num_channels=None, stream_index=None),
'english_transcription': Value('string'),
'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']),
'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']),
'path': Value('string'),
'transcription': Value('string')}
>>> ds = ds.cast_column("audio", Audio(sampling_rate=16000))
>>> ds.features
{'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None),
{'audio': Audio(sampling_rate=16000, decode=True, num_channels=None, stream_index=None),
'english_transcription': Value('string'),
'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']),
'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']),
Expand Down Expand Up @@ -4377,7 +4377,7 @@ def decode(self, enable: bool = True, num_threads: int = 0) -> "IterableDataset"
'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.'}
>>> ds = ds.decode(False)
>>> ds.features
{'image': Image(mode=None, decode=False, id=None),
{'image': Image(mode=None, decode=False),
'text': Value('string')}
>>> next(iter(ds))
{
Expand Down