Archives
/
openai-cookbook
mirror of https://github.com/openai/openai-cookbook
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enhancements and Refactoring of Python Code Extraction Methods (#467)

Browse Source 
* Refactor and enhance code extraction methods.

* Use f-strings to print filepaths, improving readability.
pull/581/head
Eli 12 months ago committed by GitHub
parent 17858f204f
commit bd91363afa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 108 additions and 82 deletions
Show Stats Download Patch File Download Diff File

190
examples/Code_search.ipynb
Unescape Escape View File

@ -7,86 +7,110 @@
"source": [
"## Code search\n",
"\n",
"We index our own [openai-python code repository](https://github.com/openai/openai-python), and show how it can be searched. We implement a simple version of file parsing and extracting of functions from python files."
"We index our own [openai-python code repository](https://github.com/openai/openai-python), and show how it can be searched. We implement a simple version of file parsing and extracting of functions from python files.\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Total number of py files: 51\n",
"Total number of functions extracted: 97\n"
"Total number of .py files: 57\n",
"Total number of functions extracted: 118\n"
]
}
],
"source": [
"import os\n",
"from glob import glob\n",
"import pandas as pd\n",
"from pathlib import Path\n",
"\n",
"DEF_PREFIXES = ['def ', 'async def ']\n",
"NEWLINE = '\\n'\n",
"\n",
"\n",
"def get_function_name(code):\n",
" \"\"\"\n",
" Extract function name from a line beginning with \"def \"\n",
" Extract function name from a line beginning with 'def' or 'async def'.\n",
" \"\"\"\n",
" assert code.startswith(\"def \")\n",
" return code[len(\"def \"): code.index(\"(\")]\n",
" for prefix in DEF_PREFIXES:\n",
" if code.startswith(prefix):\n",
" return code[len(prefix): code.index('(')]\n",
"\n",
"\n",
"def get_until_no_space(all_lines, i) -> str:\n",
"def get_until_no_space(all_lines, i):\n",
" \"\"\"\n",
" Get all lines until a line outside the function definition is found.\n",
" \"\"\"\n",
" ret = [all_lines[i]]\n",
" for j in range(i + 1, i + 10000):\n",
" if j < len(all_lines):\n",
" if len(all_lines[j]) == 0 or all_lines[j][0] in [\" \", \"\\t\", \")\"]:\n",
" ret.append(all_lines[j])\n",
" else:\n",
" break\n",
" return \"\\n\".join(ret)\n",
" for j in range(i + 1, len(all_lines)):\n",
" if len(all_lines[j]) == 0 or all_lines[j][0] in [' ', '\\t', ')']:\n",
" ret.append(all_lines[j])\n",
" else:\n",
" break\n",
" return NEWLINE.join(ret)\n",
"\n",
"\n",
"def get_functions(filepath):\n",
" \"\"\"\n",
" Get all functions in a Python file.\n",
" \"\"\"\n",
" whole_code = open(filepath).read().replace(\"\\r\", \"\\n\")\n",
" all_lines = whole_code.split(\"\\n\")\n",
" for i, l in enumerate(all_lines):\n",
" if l.startswith(\"def \"):\n",
" code = get_until_no_space(all_lines, i)\n",
" function_name = get_function_name(code)\n",
" yield {\"code\": code, \"function_name\": function_name, \"filepath\": filepath}\n",
" with open(filepath, 'r') as file:\n",
" all_lines = file.read().replace('\\r', NEWLINE).split(NEWLINE)\n",
" for i, l in enumerate(all_lines):\n",
" for prefix in DEF_PREFIXES:\n",
" if l.startswith(prefix):\n",
" code = get_until_no_space(all_lines, i)\n",
" function_name = get_function_name(code)\n",
" yield {\n",
" 'code': code,\n",
" 'function_name': function_name,\n",
" 'filepath': filepath,\n",
" }\n",
" break\n",
"\n",
"\n",
"def extract_functions_from_repo(code_root):\n",
" \"\"\"\n",
" Extract all .py functions from the repository.\n",
" \"\"\"\n",
" code_files = list(code_root.glob('**/*.py'))\n",
"\n",
" num_files = len(code_files)\n",
" print(f'Total number of .py files: {num_files}')\n",
"\n",
" if num_files == 0:\n",
" print('Verify openai-python repo exists and code_root is set correctly.')\n",
" return None\n",
"\n",
"# get user root directory\n",
"root_dir = os.path.expanduser(\"~\")\n",
"# note: for this code to work, the openai-python repo must be downloaded and placed in your root directory\n",
" all_funcs = [\n",
" func\n",
" for code_file in code_files\n",
" for func in get_functions(str(code_file))\n",
" ]\n",
"\n",
"# path to code repository directory\n",
"code_root = root_dir + \"/openai-python\"\n",
" num_funcs = len(all_funcs)\n",
" print(f'Total number of functions extracted: {num_funcs}')\n",
"\n",
"code_files = [y for x in os.walk(code_root) for y in glob(os.path.join(x[0], '*.py'))]\n",
"print(\"Total number of py files:\", len(code_files))\n",
" return all_funcs\n",
"\n",
"if len(code_files) == 0:\n",
" print(\"Double check that you have downloaded the openai-python repo and set the code_root variable correctly.\")\n",
"\n",
"all_funcs = []\n",
"for code_file in code_files:\n",
" funcs = list(get_functions(code_file))\n",
" for func in funcs:\n",
" all_funcs.append(func)\n",
"# Set user root directory to the 'openai-python' repository\n",
"root_dir = Path.home()\n",
"\n",
"print(\"Total number of functions extracted:\", len(all_funcs))"
"# Assumes the 'openai-python' repository exists in the user's root directory\n",
"code_root = root_dir / 'openai-python'\n",
"\n",
"# Extract all functions from the repository\n",
"all_funcs = extract_functions_from_repo(code_root)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 11,
"metadata": {},
"outputs": [
{
@ -121,36 +145,36 @@
" <th>0</th>\n",
" <td>def _console_log_level():\\n if openai.log i...</td>\n",
" <td>_console_log_level</td>\n",
" <td>/openai/util.py</td>\n",
" <td>[0.03389773145318031, -0.004390408284962177, 0...</td>\n",
" <td>openai/util.py</td>\n",
" <td>[0.033906757831573486, -0.00418944051489234, 0...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>def log_debug(message, **params):\\n msg = l...</td>\n",
" <td>log_debug</td>\n",
" <td>/openai/util.py</td>\n",
" <td>[-0.004034275189042091, 0.004895383026450872, ...</td>\n",
" <td>openai/util.py</td>\n",
" <td>[-0.004059609025716782, 0.004895503632724285, ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>def log_info(message, **params):\\n msg = lo...</td>\n",
" <td>log_info</td>\n",
" <td>/openai/util.py</td>\n",
" <td>[0.004882764536887407, 0.0033515947870910168, ...</td>\n",
" <td>openai/util.py</td>\n",
" <td>[0.0048639848828315735, 0.0033139237202703953,...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>def log_warn(message, **params):\\n msg = lo...</td>\n",
" <td>log_warn</td>\n",
" <td>/openai/util.py</td>\n",
" <td>[0.002535992069169879, -0.010829543694853783, ...</td>\n",
" <td>openai/util.py</td>\n",
" <td>[0.0024026145692914724, -0.010721310041844845,...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>def logfmt(props):\\n def fmt(key, val):\\n ...</td>\n",
" <td>logfmt</td>\n",
" <td>/openai/util.py</td>\n",
" <td>[0.016732551157474518, 0.017367802560329437, 0...</td>\n",
" <td>openai/util.py</td>\n",
" <td>[0.01664826273918152, 0.01730910874903202, 0.0...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@ -164,15 +188,15 @@
"3 def log_warn(message, **params):\\n msg = lo... log_warn \n",
"4 def logfmt(props):\\n def fmt(key, val):\\n ... logfmt \n",
"\n",
" filepath code_embedding \n",
"0 /openai/util.py [0.03389773145318031, -0.004390408284962177, 0... \n",
"1 /openai/util.py [-0.004034275189042091, 0.004895383026450872, ... \n",
"2 /openai/util.py [0.004882764536887407, 0.0033515947870910168, ... \n",
"3 /openai/util.py [0.002535992069169879, -0.010829543694853783, ... \n",
"4 /openai/util.py [0.016732551157474518, 0.017367802560329437, 0... "
" filepath code_embedding \n",
"0 openai/util.py [0.033906757831573486, -0.00418944051489234, 0... \n",
"1 openai/util.py [-0.004059609025716782, 0.004895503632724285, ... \n",
"2 openai/util.py [0.0048639848828315735, 0.0033139237202703953,... \n",
"3 openai/util.py [0.0024026145692914724, -0.010721310041844845,... \n",
"4 openai/util.py [0.01664826273918152, 0.01730910874903202, 0.0... "
]
},
"execution_count": 2,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
@ -182,41 +206,41 @@
"\n",
"df = pd.DataFrame(all_funcs)\n",
"df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, engine='text-embedding-ada-002'))\n",
"df['filepath'] = df['filepath'].apply(lambda x: x.replace(code_root, \"\"))\n",
"df['filepath'] = df['filepath'].map(lambda x: Path(x).relative_to(code_root))\n",
"df.to_csv(\"data/code_search_openai-python.csv\", index=False)\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/openai/tests/test_endpoints.py:test_completions score=0.826\n",
"openai/tests/test_endpoints.py:test_completions score=0.826\n",
"def test_completions():\n",
" result = openai.Completion.create(prompt=\"This was a test\", n=5, engine=\"ada\")\n",
" assert len(result.choices) == 5\n",
"\n",
"\n",
"----------------------------------------------------------------------\n",
"/openai/tests/test_endpoints.py:test_completions_model score=0.811\n",
"def test_completions_model():\n",
" result = openai.Completion.create(prompt=\"This was a test\", n=5, model=\"ada\")\n",
"openai/tests/asyncio/test_endpoints.py:test_completions score=0.824\n",
"async def test_completions():\n",
" result = await openai.Completion.acreate(\n",
" prompt=\"This was a test\", n=5, engine=\"ada\"\n",
" )\n",
" assert len(result.choices) == 5\n",
" assert result.model.startswith(\"ada\")\n",
"\n",
"\n",
"----------------------------------------------------------------------\n",
"/openai/tests/test_endpoints.py:test_completions_multiple_prompts score=0.808\n",
"def test_completions_multiple_prompts():\n",
" result = openai.Completion.create(\n",
" prompt=[\"This was a test\", \"This was another test\"], n=5, engine=\"ada\"\n",
" )\n",
" assert len(result.choices) == 10\n",
"openai/tests/asyncio/test_endpoints.py:test_completions_model score=0.82\n",
"async def test_completions_model():\n",
" result = await openai.Completion.acreate(prompt=\"This was a test\", n=5, model=\"ada\")\n",
" assert len(result.choices) == 5\n",
" assert result.model.startswith(\"ada\")\n",
"\n",
"\n",
"----------------------------------------------------------------------\n"
@ -231,11 +255,13 @@
" df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))\n",
"\n",
" res = df.sort_values('similarities', ascending=False).head(n)\n",
"\n",
" if pprint:\n",
" for r in res.iterrows():\n",
" print(r[1].filepath+\":\"+r[1].function_name + \" score=\" + str(round(r[1].similarities, 3)))\n",
" print(f\"{r[1].filepath}:{r[1].function_name} score={round(r[1].similarities, 3)}\")\n",
" print(\"\\n\".join(r[1].code.split(\"\\n\")[:n_lines]))\n",
" print('-'*70)\n",
" print('-' * 70)\n",
"\n",
" return res\n",
"\n",
"res = search_functions(df, 'Completions API tests', n=3)"
@ -243,14 +269,14 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/openai/validators.py:format_inferrer_validator score=0.751\n",
"openai/validators.py:format_inferrer_validator score=0.751\n",
"def format_inferrer_validator(df):\n",
" \"\"\"\n",
" This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n",
@ -259,7 +285,7 @@
" ft_type = infer_task_type(df)\n",
" immediate_msg = None\n",
"----------------------------------------------------------------------\n",
"/openai/validators.py:get_validators score=0.748\n",
"openai/validators.py:get_validators score=0.748\n",
"def get_validators():\n",
" return [\n",
" num_examples_validator,\n",
@ -268,7 +294,7 @@
" additional_column_validator,\n",
" non_empty_field_validator,\n",
"----------------------------------------------------------------------\n",
"/openai/validators.py:infer_task_type score=0.738\n",
"openai/validators.py:infer_task_type score=0.739\n",
"def infer_task_type(df):\n",
" \"\"\"\n",
" Infer the likely fine-tuning task type from the data\n",
@ -286,14 +312,14 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/openai/validators.py:get_common_xfix score=0.793\n",
"openai/validators.py:get_common_xfix score=0.794\n",
"def get_common_xfix(series, xfix=\"suffix\"):\n",
" \"\"\"\n",
" Finds the longest common suffix or prefix of all the values in a series\n",
@ -305,7 +331,7 @@
" if xfix == \"suffix\"\n",
" else series.str[: len(common_xfix) + 1]\n",
"----------------------------------------------------------------------\n",
"/openai/validators.py:common_completion_suffix_validator score=0.778\n",
"openai/validators.py:common_completion_suffix_validator score=0.778\n",
"def common_completion_suffix_validator(df):\n",
" \"\"\"\n",
" This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.\n",
@ -326,14 +352,14 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/openai/cli.py:tools_register score=0.773\n",
"openai/cli.py:tools_register score=0.78\n",
"def tools_register(parser):\n",
" subparsers = parser.add_subparsers(\n",
" title=\"Tools\", help=\"Convenience client side tools\"\n",
@ -382,7 +408,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.6"
"version": "3.9.16"
},
"orig_nbformat": 4
},

Write Preview
Loading…
Cancel
Save