ファイルのリストを受け取って何らかの並列処理をする。#
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from functools import partial
from pathlib import Path
from typing import List
def list_file_paths(dir_path: str) -> List[str]:
"""
List file paths in a directory.
Parameters
----------
dir_path : str
Path of the directory
Returns
-------
List[str]
List of the file paths in the directory
"""
return sorted([str(path) for path in Path(dir_path).rglob("*") if path.is_file()])
def task(same_value, same_value2, diffent_values):
return None
def multiprocess_task(task, same_value, same_value2, diffent_values):
with ProcessPoolExecutor() as executor:
results = list(executor.map(
partial(task, same_value, same_value2),
different_values
))
def multithreading_task(task, same_value, same_value2, diffent_values):
with ThreadPoolExecutor() as executor:
results = list(executor.map(
partial(task, same_value, same_value2),
different_values
))
def joblib_multiprocess_task(task : function, same_value, same_value2, diffent_values):
results = joblib.Parallel(n_jobs=-1)(
joblib.delayed(partial(task, same_value, same_value2))(different_value) for different_value in different_values)
Related Posts