I am trying to dowloand nltk data in AWS lambda from python code. But It says
{
"errorMessage": "[Errno 30] Read-only file system: 'layers'",
"errorType": "OSError",
"requestId": "",
"stackTrace": [
" File "/var/lang/lib/python3.9/importlib/__init__.py", line 127, in import_modulen return _bootstrap._gcd_import(name[level:], package, level)n",
" File "<frozen importlib._bootstrap>", line 1030, in _gcd_importn",
" File "<frozen importlib._bootstrap>", line 1007, in _find_and_loadn",
" File "<frozen importlib._bootstrap>", line 986, in _find_and_load_unlockedn",
" File "<frozen importlib._bootstrap>", line 680, in _load_unlockedn",
" File "<frozen importlib._bootstrap_external>", line 850, in exec_modulen",
" File "<frozen importlib._bootstrap>", line 228, in _call_with_frames_removedn",
" File "/opt/python/lib/python3.9/site-packages/datadog_lambda/handler.py", line 30, in <module>n handler_module = import_module(modified_mod_name)n",
" File "/var/lang/lib/python3.9/importlib/__init__.py", line 127, in import_modulen return _bootstrap._gcd_import(name[level:], package, level)n",
" File "<frozen importlib._bootstrap>", line 1030, in _gcd_importn",
" File "<frozen importlib._bootstrap>", line 1007, in _find_and_loadn",
" File "<frozen importlib._bootstrap>", line 972, in _find_and_load_unlockedn",
" File "<frozen importlib._bootstrap>", line 228, in _call_with_frames_removedn",
" File "<frozen importlib._bootstrap>", line 1030, in _gcd_importn",
" File "<frozen importlib._bootstrap>", line 1007, in _find_and_loadn",
" File "<frozen importlib._bootstrap>", line 986, in _find_and_load_unlockedn",
" File "<frozen importlib._bootstrap>", line 680, in _load_unlockedn",
" File "<frozen importlib._bootstrap_external>", line 850, in exec_modulen",
" File "<frozen importlib._bootstrap>", line 228, in _call_with_frames_removedn",
" File "/var/task/entrypoints/__init__.py", line 14, in <module>n from utils.container import state_machinen",
" File "/var/task/utils/container.py", line 10, in <module>n from service_layer.models.nltk_downloader import is_nltk_data_downloadn",
" File "/var/task/service_layer/models/nltk_downloader.py", line 12, in <module>n nltk.download("punkt", download_dir=environ["LAMBDA_NLTK_DATA"])n",
" File "/var/task/nltk/downloader.py", line 777, in downloadn for msg in self.incr_download(info_or_id, download_dir, force):n",
" File "/var/task/nltk/downloader.py", line 642, in incr_downloadn yield from self._download_package(info, download_dir, force)n",
" File "/var/task/nltk/downloader.py", line 699, in _download_packagen os.makedirs(download_dir)n",
" File "/var/lang/lib/python3.9/os.py", line 215, in makedirsn makedirs(head, exist_ok=exist_ok)n",
" File "/var/lang/lib/python3.9/os.py", line 225, in makedirsn mkdir(name, mode)n"
]
}
Here is my code
nltk.download("punkt", download_dir=download_dir)
nltk.download("wordnet", download_dir=download_dir)
nltk.download("omw-1.4", download_dir=download_dir)
I tried with a different download_dir option. But Nothing worked. For example,
download_dir = /layers/nltk_data
download_dir = nltk_data
etc
Any idea?
2
Answers
There is ephemeral storage in
/tmp
available for the lambda. For more permanent storage, you may consider writing to S3 or EFS.You can only write to folder
/tmp
. It’s default is 512Mb, but you can increase it to 10Gb in General configuration.But it isn’t good idea for storage large file, because every time cold start, it need download again.
You can try use File system, Mount the efs for lambda