|
|
@ -150,6 +150,8 @@
|
|
|
|
"metadata": {},
|
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"outputs": [],
|
|
|
|
"source": [
|
|
|
|
"source": [
|
|
|
|
|
|
|
|
"import os\n",
|
|
|
|
|
|
|
|
"\n",
|
|
|
|
"import arxiv\n",
|
|
|
|
"import arxiv\n",
|
|
|
|
"import ast\n",
|
|
|
|
"import ast\n",
|
|
|
|
"import concurrent\n",
|
|
|
|
"import concurrent\n",
|
|
|
@ -190,6 +192,25 @@
|
|
|
|
"id": "2de5d32d",
|
|
|
|
"id": "2de5d32d",
|
|
|
|
"metadata": {},
|
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"outputs": [],
|
|
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
|
|
"directory = './data/papers'\n",
|
|
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
|
|
"# Check if the directory already exists\n",
|
|
|
|
|
|
|
|
"if not os.path.exists(directory):\n",
|
|
|
|
|
|
|
|
" # If the directory doesn't exist, create it and any necessary intermediate directories\n",
|
|
|
|
|
|
|
|
" os.makedirs(directory)\n",
|
|
|
|
|
|
|
|
" print(f\"Directory '{directory}' created successfully.\")\n",
|
|
|
|
|
|
|
|
"else:\n",
|
|
|
|
|
|
|
|
" # If the directory already exists, print a message indicating it\n",
|
|
|
|
|
|
|
|
" print(f\"Directory '{directory}' already exists.\")"
|
|
|
|
|
|
|
|
]
|
|
|
|
|
|
|
|
},
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
|
|
"execution_count": null,
|
|
|
|
|
|
|
|
"id": "ae5cb7a1",
|
|
|
|
|
|
|
|
"metadata": {},
|
|
|
|
|
|
|
|
"outputs": [],
|
|
|
|
"source": [
|
|
|
|
"source": [
|
|
|
|
"# Set a directory to store downloaded papers\n",
|
|
|
|
"# Set a directory to store downloaded papers\n",
|
|
|
|
"data_dir = os.path.join(os.curdir, \"data\", \"papers\")\n",
|
|
|
|
"data_dir = os.path.join(os.curdir, \"data\", \"papers\")\n",
|
|
|
@ -197,7 +218,7 @@
|
|
|
|
"\n",
|
|
|
|
"\n",
|
|
|
|
"# Generate a blank dataframe where we can store downloaded files\n",
|
|
|
|
"# Generate a blank dataframe where we can store downloaded files\n",
|
|
|
|
"df = pd.DataFrame(list())\n",
|
|
|
|
"df = pd.DataFrame(list())\n",
|
|
|
|
"df.to_csv(paper_dir_filepath)\n"
|
|
|
|
"df.to_csv(paper_dir_filepath)"
|
|
|
|
]
|
|
|
|
]
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
{
|
|
|
|