# Map with num\_proc over 1 fails

**URL:** </static-proxy?url=https%3A%2F%2Fdiscuss.huggingface.co%2Ft%2Fmap-with-num-proc-over-1-fails%2F83049%3E%5C
**Category:** 🤗Datasets\
**Created:** [April 22, 2024, 7:53am UTC](/static-proxy?url=https%3A%2F%2Fdiscuss.huggingface.co%2Ft%2Fmap-with-num-proc-over-1-fails%2F83049 "2024-04-22T07:53:01Z")\
**Posts on this page:** 2\
**Page:** 1

<div class="post-metadata">

**Author:** ![jason-kim](https://avatars.discourse-cdn.com/v4/letter/j/4491bb/32.png) [@jason-kim](/static-proxy?url=https%3A%2F%2Fdiscuss.huggingface.co%2Fu%2Fjason-kim)%5C
**Post date:** [April 22, 2024, 7:53am UTC](/static-proxy?url=https%3A%2F%2Fdiscuss.huggingface.co%2Ft%2Fmap-with-num-proc-over-1-fails%2F83049%2F1 "2024-04-22T07:53:01Z")

</div>

Hi,  
I have tested with simple custom text data.  
map with num\_proc of 1 or none is fine but num\_proc over 1 occurs PermissionError.

```auto
from datasets import load_dataset

def preprocess(examples):
    ex = [ex.replace('design', 'text') for ex in examples["name"]]
    examples["name"] = ex
    return examples

train_file = {"train":"test.csv"}
ds_raw = load_dataset("csv", data_files=train_file, sep="\t")
ds = ds_raw.map(preprocess, batched=True, num_proc=2)

```

* * *

> PermissionError Traceback (most recent call last)  
> Cell In[66], line 1  
> ----\> 1 ds = ds\_raw.map(preprocess, batched=True, num\_proc=2)
> 
> File ~/.venv/lib/python3.10/site-packages/datasets/dataset\_dict.py:868, in DatasetDict.map(self, function, with\_indices, with\_rank, input\_columns, batched, batch\_size, drop\_last\_batch, remove\_columns, keep\_in\_memory, load\_from\_cache\_file, cache\_file\_names, writer\_batch\_size, features, disable\_nullable, fn\_kwargs, num\_proc, desc)  
> 865 if cache\_file\_names is None:  
> 866 cache\_file\_names = {k: None for k in self}  
> 867 return DatasetDict(  
> → 868 {  
> 869 k: dataset.map(  
> 870 function=function,  
> 871 with\_indices=with\_indices,  
> 872 with\_rank=with\_rank,  
> 873 input\_columns=input\_columns,  
> 874 batched=batched,  
> 875 batch\_size=batch\_size,  
> 876 drop\_last\_batch=drop\_last\_batch,  
> 877 remove\_columns=remove\_columns,  
> 878 keep\_in\_memory=keep\_in\_memory,  
> 879 load\_from\_cache\_file=load\_from\_cache\_file,  
> 880 cache\_file\_name=cache\_file\_names[k],  
> 881 writer\_batch\_size=writer\_batch\_size,  
> 882 features=features,  
> 883 disable\_nullable=disable\_nullable,  
> 884 fn\_kwargs=fn\_kwargs,  
> 885 num\_proc=num\_proc,  
> 886 desc=desc,  
> 887 )  
> 888 for k, dataset in self.items()  
> 889 }  
> 890 )
> 
> File ~/.venv/lib/python3.10/site-packages/datasets/dataset\_dict.py:869, in (.0)  
> 865 if cache\_file\_names is None:  
> 866 cache\_file\_names = {k: None for k in self}  
> 867 return DatasetDict(  
> 868 {  
> → 869 k: dataset.map(  
> 870 function=function,  
> 871 with\_indices=with\_indices,  
> 872 with\_rank=with\_rank,  
> 873 input\_columns=input\_columns,  
> 874 batched=batched,  
> 875 batch\_size=batch\_size,  
> 876 drop\_last\_batch=drop\_last\_batch,  
> 877 remove\_columns=remove\_columns,  
> 878 keep\_in\_memory=keep\_in\_memory,  
> 879 load\_from\_cache\_file=load\_from\_cache\_file,  
> 880 cache\_file\_name=cache\_file\_names[k],  
> 881 writer\_batch\_size=writer\_batch\_size,  
> 882 features=features,  
> 883 disable\_nullable=disable\_nullable,  
> 884 fn\_kwargs=fn\_kwargs,  
> 885 num\_proc=num\_proc,  
> 886 desc=desc,  
> 887 )  
> 888 for k, dataset in self.items()  
> 889 }  
> 890 )
> 
> File ~/.venv/lib/python3.10/site-packages/datasets/arrow\_dataset.py:592, in transmit\_tasks..wrapper(\*args, \*\*kwargs)  
> 590 self: “Dataset” = kwargs.pop(“self”)  
> 591 # apply actual function  
> → 592 out: Union[“Dataset”, “DatasetDict”] = func(self, \*args, \*\*kwargs)  
> 593 datasets: List[“Dataset”] = list(out.values()) if isinstance(out, dict) else [out]  
> 594 for dataset in datasets:  
> 595 # Remove task templates if a column mapping of the template is no longer valid
> 
> File ~/.venv/lib/python3.10/site-packages/datasets/arrow\_dataset.py:557, in transmit\_format..wrapper(\*args, \*\*kwargs)  
> 550 self\_format = {  
> 551 “type”: self.\_format\_type,  
> 552 “format\_kwargs”: self.\_format\_kwargs,  
> 553 “columns”: self.\_format\_columns,  
> 554 “output\_all\_columns”: self.\_output\_all\_columns,  
> 555 }  
> 556 # apply actual function  
> → 557 out: Union[“Dataset”, “DatasetDict”] = func(self, \*args, \*\*kwargs)  
> 558 datasets: List[“Dataset”] = list(out.values()) if isinstance(out, dict) else [out]  
> 559 # re-apply format to the output
> 
> File ~/.venv/lib/python3.10/site-packages/datasets/arrow\_dataset.py:3177, in Dataset.map(self, function, with\_indices, with\_rank, input\_columns, batched, batch\_size, drop\_last\_batch, remove\_columns, keep\_in\_memory, load\_from\_cache\_file, cache\_file\_name, writer\_batch\_size, features, disable\_nullable, fn\_kwargs, num\_proc, suffix\_template, new\_fingerprint, desc)  
> 3173 if len(kwargs\_per\_job) \< num\_shards:  
> 3174 logger.info(  
> 3175 f"Reprocessing {len(kwargs\_per\_job)}/{num\_shards} shards because some of them were missing from the cache."  
> 3176 )  
> → 3177 with Pool(len(kwargs\_per\_job)) as pool:  
> 3178 os.environ = prev\_env  
> 3179 logger.info(f"Spawning {num\_proc} processes")
> 
> File ~/.venv/lib/python3.10/site-packages/multiprocess/context.py:119, in BaseContext.Pool(self, processes, initializer, initargs, maxtasksperchild)  
> 117 ‘’‘Returns a process pool object’‘’  
> 118 from .pool import Pool  
> → 119 return Pool(processes, initializer, initargs, maxtasksperchild,  
> 120 context=self.get\_context())
> 
> File ~/.venv/lib/python3.10/site-packages/multiprocess/pool.py:191, in Pool. **init** (self, processes, initializer, initargs, maxtasksperchild, context)  
> 188 self.\_state = INIT  
> 190 self.\_ctx = context or get\_context()  
> → 191 self.\_setup\_queues()  
> 192 self.\_taskqueue = queue.SimpleQueue()  
> 193 # The \_change\_notifier queue exist to wake up self.\_handle\_workers()  
> 194 # when the cache (self.\_cache) is empty or when there is a change in  
> 195 # the \_state variable of the thread that runs \_handle\_workers.
> 
> File ~/.venv/lib/python3.10/site-packages/multiprocess/pool.py:346, in Pool.\_setup\_queues(self)  
> 345 def \_setup\_queues(self):  
> → 346 self.\_inqueue = self.\_ctx.SimpleQueue()  
> 347 self.\_outqueue = self.\_ctx.SimpleQueue()  
> 348 self.\_quick\_put = self.\_inqueue.\_writer.send
> 
> File ~/.venv/lib/python3.10/site-packages/multiprocess/context.py:113, in BaseContext.SimpleQueue(self)  
> 111 ‘’‘Returns a queue object’‘’  
> 112 from .queues import SimpleQueue  
> → 113 return SimpleQueue(ctx=self.get\_context())
> 
> File ~/.venv/lib/python3.10/site-packages/multiprocess/queues.py:344, in SimpleQueue. **init** (self, ctx)  
> 342 def **init** (self, \*, ctx):  
> 343 self.\_reader, self.\_writer = connection.Pipe(duplex=False)  
> → 344 self.\_rlock = ctx.Lock()  
> 345 self.\_poll = self.\_reader.poll  
> 346 if sys.platform == ‘win32’:
> 
> File ~/.venv/lib/python3.10/site-packages/multiprocess/context.py:68, in BaseContext.Lock(self)  
> 66 ‘’‘Returns a non-recursive lock object’‘’  
> 67 from .synchronize import Lock  
> —\> 68 return Lock(ctx=self.get\_context())
> 
> File ~/.venv/lib/python3.10/site-packages/multiprocess/synchronize.py:168, in Lock. **init** (self, ctx)  
> 167 def **init** (self, \*, ctx):  
> → 168 SemLock. **init** (self, SEMAPHORE, 1, 1, ctx=ctx)
> 
> File ~/.venv/lib/python3.10/site-packages/multiprocess/synchronize.py:63, in SemLock. **init** (self, kind, value, maxvalue, ctx)  
> 61 for i in range(100):  
> 62 try:  
> —\> 63 sl = self.\_semlock = \_multiprocessing.SemLock(  
> 64 kind, value, maxvalue, self.\_make\_name(),  
> 65 unlink\_now)  
> 66 except FileExistsError:  
> 67 pass
> 
> PermissionError: [Errno 13] Permission denied

---

<div class="post-metadata">

**Author:** ![jason-kim](https://avatars.discourse-cdn.com/v4/letter/j/4491bb/32.png) [@jason-kim](/static-proxy?url=https%3A%2F%2Fdiscuss.huggingface.co%2Fu%2Fjason-kim)%5C
**Post date:** [April 24, 2024, 5:24am UTC](/static-proxy?url=https%3A%2F%2Fdiscuss.huggingface.co%2Ft%2Fmap-with-num-proc-over-1-fails%2F83049%2F2 "2024-04-24T05:24:59Z")

</div>

I solved it with [linux - Python multiprocessing: Permission denied - Stack Overflow](https://stackoverflow.com/a/2009505/4910342)