@@ -347,6 +347,7 @@ class ParallelProcessing(Generic[_Target_P, _Target_T, _Dataset_T]):
347347 """
348348
349349 _length : int
350+ _retrieve_value : Callable [[Any , int ], _Dataset_T ]
350351 _threads : List [_ThreadWorker ]
351352 _completed : int
352353
@@ -484,6 +485,7 @@ def __init__(
484485 assert get_value , '`_get_value` must be set'
485486
486487 self ._length = length
488+ self ._retrieve_value = get_value
487489 self ._threads = []
488490 self ._completed = 0
489491
@@ -613,7 +615,7 @@ def start(self) -> None:
613615 raise exceptions .ThreadStillRunningError ()
614616
615617 self .status = 'Running'
616- max_threads = min (self .max_threads , len ( self .dataset ) )
618+ max_threads = min (self .max_threads , self ._length )
617619
618620 parsed_args = self .overflow_kwargs .get ('args' , [])
619621 name_format = (
@@ -624,13 +626,16 @@ def start(self) -> None:
624626 }
625627
626628 i = 0
627- for chunkStart , chunkEnd in chunk_split (len ( self .dataset ) , max_threads ):
629+ for chunkStart , chunkEnd in chunk_split (self ._length , max_threads ):
628630 chunk_thread = Thread (
629631 target = self .function ,
630632 args = [
631633 i ,
632634 chunkEnd - chunkStart ,
633- (self .dataset [x ] for x in range (chunkStart , chunkEnd )),
635+ (
636+ self ._retrieve_value (self .dataset , x )
637+ for x in range (chunkStart , chunkEnd )
638+ ),
634639 * parsed_args ,
635640 * self .overflow_args ,
636641 ],
0 commit comments