mirror of
https://github.com/hwchase17/langchain
synced 2024-11-02 09:40:22 +00:00
docs: arxiv page, added cookbooks (#22215)
Issue: The `arXiv` page is missing the arxiv paper references from the `langchain/cookbook`. PR: Added the cookbook references. Result: `Found 29 arXiv references in the 3 docs, 21 API Refs, 5 Templates, and 18 Cookbooks.` - much more references are visible now.
This commit is contained in:
Leonid Ganeline
GitHub
parent
d6995e814b
commit
d7f70535ba
@ -2,32 +2,150 @@
|
||||
|
||||
LangChain implements the latest research in the field of Natural Language Processing.
|
||||
This page contains `arXiv` papers referenced in the LangChain Documentation, API Reference,
|
||||
and Templates.
|
||||
Templates, and Cookbooks.
|
||||
|
||||
## Summary
|
||||
|
||||
| arXiv id / Title | Authors | Published date 🔻 | LangChain Documentation|
|
||||
|------------------|---------|-------------------|------------------------|
|
||||
| `2402.03620v1` [Self-Discover: Large Language Models Self-Compose Reasoning Structures](http://arxiv.org/abs/2402.03620v1) | Pei Zhou, Jay Pujara, Xiang Ren, et al. | 2024-02-06 | `Cookbook:` [self-discover](https://github.com/langchain-ai/langchain/blob/master/cookbook/self-discover.ipynb)
|
||||
| `2401.18059v1` [RAPTOR: Recursive Abstractive Processing for Tree-Organized Retrieval](http://arxiv.org/abs/2401.18059v1) | Parth Sarthi, Salman Abdullah, Aditi Tuli, et al. | 2024-01-31 | `Cookbook:` [RAPTOR](https://github.com/langchain-ai/langchain/blob/master/cookbook/RAPTOR.ipynb)
|
||||
| `2401.15884v2` [Corrective Retrieval Augmented Generation](http://arxiv.org/abs/2401.15884v2) | Shi-Qi Yan, Jia-Chen Gu, Yun Zhu, et al. | 2024-01-29 | `Cookbook:` [langgraph_crag](https://github.com/langchain-ai/langchain/blob/master/cookbook/langgraph_crag.ipynb)
|
||||
| `2401.04088v1` [Mixtral of Experts](http://arxiv.org/abs/2401.04088v1) | Albert Q. Jiang, Alexandre Sablayrolles, Antoine Roux, et al. | 2024-01-08 | `Cookbook:` [together_ai](https://github.com/langchain-ai/langchain/blob/master/cookbook/together_ai.ipynb)
|
||||
| `2312.06648v2` [Dense X Retrieval: What Retrieval Granularity Should We Use?](http://arxiv.org/abs/2312.06648v2) | Tong Chen, Hongwei Wang, Sihao Chen, et al. | 2023-12-11 | `Template:` [propositional-retrieval](https://python.langchain.com/docs/templates/propositional-retrieval)
|
||||
| `2311.09210v1` [Chain-of-Note: Enhancing Robustness in Retrieval-Augmented Language Models](http://arxiv.org/abs/2311.09210v1) | Wenhao Yu, Hongming Zhang, Xiaoman Pan, et al. | 2023-11-15 | `Template:` [chain-of-note-wiki](https://python.langchain.com/docs/templates/chain-of-note-wiki)
|
||||
| `2310.06117v2` [Take a Step Back: Evoking Reasoning via Abstraction in Large Language Models](http://arxiv.org/abs/2310.06117v2) | Huaixiu Steven Zheng, Swaroop Mishra, Xinyun Chen, et al. | 2023-10-09 | `Template:` [stepback-qa-prompting](https://python.langchain.com/docs/templates/stepback-qa-prompting)
|
||||
| `2305.14283v3` [Query Rewriting for Retrieval-Augmented Large Language Models](http://arxiv.org/abs/2305.14283v3) | Xinbei Ma, Yeyun Gong, Pengcheng He, et al. | 2023-05-23 | `Template:` [rewrite-retrieve-read](https://python.langchain.com/docs/templates/rewrite-retrieve-read)
|
||||
| `2305.08291v1` [Large Language Model Guided Tree-of-Thought](http://arxiv.org/abs/2305.08291v1) | Jieyi Long | 2023-05-15 | `API:` [langchain_experimental.tot](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.tot)
|
||||
| `2303.17580v4` [HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in Hugging Face](http://arxiv.org/abs/2303.17580v4) | Yongliang Shen, Kaitao Song, Xu Tan, et al. | 2023-03-30 | `API:` [langchain_experimental.autonomous_agents](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.autonomous_agents)
|
||||
| `2310.11511v1` [Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection](http://arxiv.org/abs/2310.11511v1) | Akari Asai, Zeqiu Wu, Yizhong Wang, et al. | 2023-10-17 | `Cookbook:` [langgraph_self_rag](https://github.com/langchain-ai/langchain/blob/master/cookbook/langgraph_self_rag.ipynb)
|
||||
| `2310.06117v2` [Take a Step Back: Evoking Reasoning via Abstraction in Large Language Models](http://arxiv.org/abs/2310.06117v2) | Huaixiu Steven Zheng, Swaroop Mishra, Xinyun Chen, et al. | 2023-10-09 | `Template:` [stepback-qa-prompting](https://python.langchain.com/docs/templates/stepback-qa-prompting), `Cookbook:` [stepback-qa](https://github.com/langchain-ai/langchain/blob/master/cookbook/stepback-qa.ipynb)
|
||||
| `2307.09288v2` [Llama 2: Open Foundation and Fine-Tuned Chat Models](http://arxiv.org/abs/2307.09288v2) | Hugo Touvron, Louis Martin, Kevin Stone, et al. | 2023-07-18 | `Cookbook:` [Semi_Structured_RAG](https://github.com/langchain-ai/langchain/blob/master/cookbook/Semi_Structured_RAG.ipynb)
|
||||
| `2305.14283v3` [Query Rewriting for Retrieval-Augmented Large Language Models](http://arxiv.org/abs/2305.14283v3) | Xinbei Ma, Yeyun Gong, Pengcheng He, et al. | 2023-05-23 | `Template:` [rewrite-retrieve-read](https://python.langchain.com/docs/templates/rewrite-retrieve-read), `Cookbook:` [rewrite](https://github.com/langchain-ai/langchain/blob/master/cookbook/rewrite.ipynb)
|
||||
| `2305.08291v1` [Large Language Model Guided Tree-of-Thought](http://arxiv.org/abs/2305.08291v1) | Jieyi Long | 2023-05-15 | `API:` [langchain_experimental.tot](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.tot), `Cookbook:` [tree_of_thought](https://github.com/langchain-ai/langchain/blob/master/cookbook/tree_of_thought.ipynb)
|
||||
| `2305.04091v3` [Plan-and-Solve Prompting: Improving Zero-Shot Chain-of-Thought Reasoning by Large Language Models](http://arxiv.org/abs/2305.04091v3) | Lei Wang, Wanyu Xu, Yihuai Lan, et al. | 2023-05-06 | `Cookbook:` [plan_and_execute_agent](https://github.com/langchain-ai/langchain/blob/master/cookbook/plan_and_execute_agent.ipynb)
|
||||
| `2304.08485v2` [Visual Instruction Tuning](http://arxiv.org/abs/2304.08485v2) | Haotian Liu, Chunyuan Li, Qingyang Wu, et al. | 2023-04-17 | `Cookbook:` [Semi_structured_and_multi_modal_RAG](https://github.com/langchain-ai/langchain/blob/master/cookbook/Semi_structured_and_multi_modal_RAG.ipynb), [Semi_structured_multi_modal_RAG_LLaMA2](https://github.com/langchain-ai/langchain/blob/master/cookbook/Semi_structured_multi_modal_RAG_LLaMA2.ipynb)
|
||||
| `2304.03442v2` [Generative Agents: Interactive Simulacra of Human Behavior](http://arxiv.org/abs/2304.03442v2) | Joon Sung Park, Joseph C. O'Brien, Carrie J. Cai, et al. | 2023-04-07 | `Cookbook:` [multiagent_bidding](https://github.com/langchain-ai/langchain/blob/master/cookbook/multiagent_bidding.ipynb), [generative_agents_interactive_simulacra_of_human_behavior](https://github.com/langchain-ai/langchain/blob/master/cookbook/generative_agents_interactive_simulacra_of_human_behavior.ipynb)
|
||||
| `2303.17760v2` [CAMEL: Communicative Agents for "Mind" Exploration of Large Language Model Society](http://arxiv.org/abs/2303.17760v2) | Guohao Li, Hasan Abed Al Kader Hammoud, Hani Itani, et al. | 2023-03-31 | `Cookbook:` [camel_role_playing](https://github.com/langchain-ai/langchain/blob/master/cookbook/camel_role_playing.ipynb)
|
||||
| `2303.17580v4` [HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in Hugging Face](http://arxiv.org/abs/2303.17580v4) | Yongliang Shen, Kaitao Song, Xu Tan, et al. | 2023-03-30 | `API:` [langchain_experimental.autonomous_agents](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.autonomous_agents), `Cookbook:` [hugginggpt](https://github.com/langchain-ai/langchain/blob/master/cookbook/hugginggpt.ipynb)
|
||||
| `2303.08774v6` [GPT-4 Technical Report](http://arxiv.org/abs/2303.08774v6) | OpenAI, Josh Achiam, Steven Adler, et al. | 2023-03-15 | `Docs:` [docs/integrations/vectorstores/mongodb_atlas](https://python.langchain.com/docs/integrations/vectorstores/mongodb_atlas)
|
||||
| `2301.10226v4` [A Watermark for Large Language Models](http://arxiv.org/abs/2301.10226v4) | John Kirchenbauer, Jonas Geiping, Yuxin Wen, et al. | 2023-01-24 | `API:` [langchain_community.llms...OCIModelDeploymentTGI](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI.html#langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI), [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
| `2212.10496v1` [Precise Zero-Shot Dense Retrieval without Relevance Labels](http://arxiv.org/abs/2212.10496v1) | Luyu Gao, Xueguang Ma, Jimmy Lin, et al. | 2022-12-20 | `API:` [langchain.chains...HypotheticalDocumentEmbedder](https://api.python.langchain.com/en/latest/chains/langchain.chains.hyde.base.HypotheticalDocumentEmbedder.html#langchain.chains.hyde.base.HypotheticalDocumentEmbedder), `Template:` [hyde](https://python.langchain.com/docs/templates/hyde)
|
||||
| `2301.10226v4` [A Watermark for Large Language Models](http://arxiv.org/abs/2301.10226v4) | John Kirchenbauer, Jonas Geiping, Yuxin Wen, et al. | 2023-01-24 | `API:` [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_huggingface.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community.llms...OCIModelDeploymentTGI](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI.html#langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI)
|
||||
| `2212.10496v1` [Precise Zero-Shot Dense Retrieval without Relevance Labels](http://arxiv.org/abs/2212.10496v1) | Luyu Gao, Xueguang Ma, Jimmy Lin, et al. | 2022-12-20 | `API:` [langchain.chains...HypotheticalDocumentEmbedder](https://api.python.langchain.com/en/latest/chains/langchain.chains.hyde.base.HypotheticalDocumentEmbedder.html#langchain.chains.hyde.base.HypotheticalDocumentEmbedder), `Template:` [hyde](https://python.langchain.com/docs/templates/hyde), `Cookbook:` [hypothetical_document_embeddings](https://github.com/langchain-ai/langchain/blob/master/cookbook/hypothetical_document_embeddings.ipynb)
|
||||
| `2212.07425v3` [Robust and Explainable Identification of Logical Fallacies in Natural Language Arguments](http://arxiv.org/abs/2212.07425v3) | Zhivar Sourati, Vishnu Priya Prasanna Venkatesh, Darshan Deshpande, et al. | 2022-12-12 | `API:` [langchain_experimental.fallacy_removal](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.fallacy_removal)
|
||||
| `2211.13892v2` [Complementary Explanations for Effective In-Context Learning](http://arxiv.org/abs/2211.13892v2) | Xi Ye, Srinivasan Iyer, Asli Celikyilmaz, et al. | 2022-11-25 | `API:` [langchain_core.example_selectors...MaxMarginalRelevanceExampleSelector](https://api.python.langchain.com/en/latest/example_selectors/langchain_core.example_selectors.semantic_similarity.MaxMarginalRelevanceExampleSelector.html#langchain_core.example_selectors.semantic_similarity.MaxMarginalRelevanceExampleSelector)
|
||||
| `2211.10435v2` [PAL: Program-aided Language Models](http://arxiv.org/abs/2211.10435v2) | Luyu Gao, Aman Madaan, Shuyan Zhou, et al. | 2022-11-18 | `API:` [langchain_experimental.pal_chain...PALChain](https://api.python.langchain.com/en/latest/pal_chain/langchain_experimental.pal_chain.base.PALChain.html#langchain_experimental.pal_chain.base.PALChain), [langchain_experimental.pal_chain](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.pal_chain)
|
||||
| `2211.10435v2` [PAL: Program-aided Language Models](http://arxiv.org/abs/2211.10435v2) | Luyu Gao, Aman Madaan, Shuyan Zhou, et al. | 2022-11-18 | `API:` [langchain_experimental.pal_chain](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.pal_chain), [langchain_experimental.pal_chain...PALChain](https://api.python.langchain.com/en/latest/pal_chain/langchain_experimental.pal_chain.base.PALChain.html#langchain_experimental.pal_chain.base.PALChain), `Cookbook:` [program_aided_language_model](https://github.com/langchain-ai/langchain/blob/master/cookbook/program_aided_language_model.ipynb)
|
||||
| `2209.10785v2` [Deep Lake: a Lakehouse for Deep Learning](http://arxiv.org/abs/2209.10785v2) | Sasun Hambardzumyan, Abhinav Tuli, Levon Ghukasyan, et al. | 2022-09-22 | `Docs:` [docs/integrations/providers/activeloop_deeplake](https://python.langchain.com/docs/integrations/providers/activeloop_deeplake)
|
||||
| `2205.12654v1` [Bitext Mining Using Distilled Sentence Representations for Low-Resource Languages](http://arxiv.org/abs/2205.12654v1) | Kevin Heffernan, Onur Çelebi, Holger Schwenk | 2022-05-25 | `API:` [langchain_community.embeddings...LaserEmbeddings](https://api.python.langchain.com/en/latest/embeddings/langchain_community.embeddings.laser.LaserEmbeddings.html#langchain_community.embeddings.laser.LaserEmbeddings)
|
||||
| `2204.00498v1` [Evaluating the Text-to-SQL Capabilities of Large Language Models](http://arxiv.org/abs/2204.00498v1) | Nitarshan Rajkumar, Raymond Li, Dzmitry Bahdanau | 2022-03-15 | `API:` [langchain_community.utilities...SQLDatabase](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.sql_database.SQLDatabase.html#langchain_community.utilities.sql_database.SQLDatabase), [langchain_community.utilities...SparkSQL](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.spark_sql.SparkSQL.html#langchain_community.utilities.spark_sql.SparkSQL)
|
||||
| `2202.00666v5` [Locally Typical Sampling](http://arxiv.org/abs/2202.00666v5) | Clara Meister, Tiago Pimentel, Gian Wiher, et al. | 2022-02-01 | `API:` [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
| `2204.00498v1` [Evaluating the Text-to-SQL Capabilities of Large Language Models](http://arxiv.org/abs/2204.00498v1) | Nitarshan Rajkumar, Raymond Li, Dzmitry Bahdanau | 2022-03-15 | `API:` [langchain_community.utilities...SparkSQL](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.spark_sql.SparkSQL.html#langchain_community.utilities.spark_sql.SparkSQL), [langchain_community.utilities...SQLDatabase](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.sql_database.SQLDatabase.html#langchain_community.utilities.sql_database.SQLDatabase)
|
||||
| `2202.00666v5` [Locally Typical Sampling](http://arxiv.org/abs/2202.00666v5) | Clara Meister, Tiago Pimentel, Gian Wiher, et al. | 2022-02-01 | `API:` [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_huggingface.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
| `2103.00020v1` [Learning Transferable Visual Models From Natural Language Supervision](http://arxiv.org/abs/2103.00020v1) | Alec Radford, Jong Wook Kim, Chris Hallacy, et al. | 2021-02-26 | `API:` [langchain_experimental.open_clip](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.open_clip)
|
||||
| `1909.05858v2` [CTRL: A Conditional Transformer Language Model for Controllable Generation](http://arxiv.org/abs/1909.05858v2) | Nitish Shirish Keskar, Bryan McCann, Lav R. Varshney, et al. | 2019-09-11 | `API:` [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
| `1909.05858v2` [CTRL: A Conditional Transformer Language Model for Controllable Generation](http://arxiv.org/abs/1909.05858v2) | Nitish Shirish Keskar, Bryan McCann, Lav R. Varshney, et al. | 2019-09-11 | `API:` [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_huggingface.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
| `1908.10084v1` [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](http://arxiv.org/abs/1908.10084v1) | Nils Reimers, Iryna Gurevych | 2019-08-27 | `Docs:` [docs/integrations/text_embedding/sentence_transformers](https://python.langchain.com/docs/integrations/text_embedding/sentence_transformers)
|
||||
|
||||
## Self-Discover: Large Language Models Self-Compose Reasoning Structures
|
||||
|
||||
- **arXiv id:** 2402.03620v1
|
||||
- **Title:** Self-Discover: Large Language Models Self-Compose Reasoning Structures
|
||||
- **Authors:** Pei Zhou, Jay Pujara, Xiang Ren, et al.
|
||||
- **Published Date:** 2024-02-06
|
||||
- **URL:** http://arxiv.org/abs/2402.03620v1
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [self-discover](https://github.com/langchain-ai/langchain/blob/master/cookbook/self-discover.ipynb)
|
||||
|
||||
**Abstract:** We introduce SELF-DISCOVER, a general framework for LLMs to self-discover the
|
||||
task-intrinsic reasoning structures to tackle complex reasoning problems that
|
||||
are challenging for typical prompting methods. Core to the framework is a
|
||||
self-discovery process where LLMs select multiple atomic reasoning modules such
|
||||
as critical thinking and step-by-step thinking, and compose them into an
|
||||
explicit reasoning structure for LLMs to follow during decoding. SELF-DISCOVER
|
||||
substantially improves GPT-4 and PaLM 2's performance on challenging reasoning
|
||||
benchmarks such as BigBench-Hard, grounded agent reasoning, and MATH, by as
|
||||
much as 32% compared to Chain of Thought (CoT). Furthermore, SELF-DISCOVER
|
||||
outperforms inference-intensive methods such as CoT-Self-Consistency by more
|
||||
than 20%, while requiring 10-40x fewer inference compute. Finally, we show that
|
||||
the self-discovered reasoning structures are universally applicable across
|
||||
model families: from PaLM 2-L to GPT-4, and from GPT-4 to Llama2, and share
|
||||
commonalities with human reasoning patterns.
|
||||
|
||||
## RAPTOR: Recursive Abstractive Processing for Tree-Organized Retrieval
|
||||
|
||||
- **arXiv id:** 2401.18059v1
|
||||
- **Title:** RAPTOR: Recursive Abstractive Processing for Tree-Organized Retrieval
|
||||
- **Authors:** Parth Sarthi, Salman Abdullah, Aditi Tuli, et al.
|
||||
- **Published Date:** 2024-01-31
|
||||
- **URL:** http://arxiv.org/abs/2401.18059v1
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [RAPTOR](https://github.com/langchain-ai/langchain/blob/master/cookbook/RAPTOR.ipynb)
|
||||
|
||||
**Abstract:** Retrieval-augmented language models can better adapt to changes in world
|
||||
state and incorporate long-tail knowledge. However, most existing methods
|
||||
retrieve only short contiguous chunks from a retrieval corpus, limiting
|
||||
holistic understanding of the overall document context. We introduce the novel
|
||||
approach of recursively embedding, clustering, and summarizing chunks of text,
|
||||
constructing a tree with differing levels of summarization from the bottom up.
|
||||
At inference time, our RAPTOR model retrieves from this tree, integrating
|
||||
information across lengthy documents at different levels of abstraction.
|
||||
Controlled experiments show that retrieval with recursive summaries offers
|
||||
significant improvements over traditional retrieval-augmented LMs on several
|
||||
tasks. On question-answering tasks that involve complex, multi-step reasoning,
|
||||
we show state-of-the-art results; for example, by coupling RAPTOR retrieval
|
||||
with the use of GPT-4, we can improve the best performance on the QuALITY
|
||||
benchmark by 20% in absolute accuracy.
|
||||
|
||||
## Corrective Retrieval Augmented Generation
|
||||
|
||||
- **arXiv id:** 2401.15884v2
|
||||
- **Title:** Corrective Retrieval Augmented Generation
|
||||
- **Authors:** Shi-Qi Yan, Jia-Chen Gu, Yun Zhu, et al.
|
||||
- **Published Date:** 2024-01-29
|
||||
- **URL:** http://arxiv.org/abs/2401.15884v2
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [langgraph_crag](https://github.com/langchain-ai/langchain/blob/master/cookbook/langgraph_crag.ipynb)
|
||||
|
||||
**Abstract:** Large language models (LLMs) inevitably exhibit hallucinations since the
|
||||
accuracy of generated texts cannot be secured solely by the parametric
|
||||
knowledge they encapsulate. Although retrieval-augmented generation (RAG) is a
|
||||
practicable complement to LLMs, it relies heavily on the relevance of retrieved
|
||||
documents, raising concerns about how the model behaves if retrieval goes
|
||||
wrong. To this end, we propose the Corrective Retrieval Augmented Generation
|
||||
(CRAG) to improve the robustness of generation. Specifically, a lightweight
|
||||
retrieval evaluator is designed to assess the overall quality of retrieved
|
||||
documents for a query, returning a confidence degree based on which different
|
||||
knowledge retrieval actions can be triggered. Since retrieval from static and
|
||||
limited corpora can only return sub-optimal documents, large-scale web searches
|
||||
are utilized as an extension for augmenting the retrieval results. Besides, a
|
||||
decompose-then-recompose algorithm is designed for retrieved documents to
|
||||
selectively focus on key information and filter out irrelevant information in
|
||||
them. CRAG is plug-and-play and can be seamlessly coupled with various
|
||||
RAG-based approaches. Experiments on four datasets covering short- and
|
||||
long-form generation tasks show that CRAG can significantly improve the
|
||||
performance of RAG-based approaches.
|
||||
|
||||
## Mixtral of Experts
|
||||
|
||||
- **arXiv id:** 2401.04088v1
|
||||
- **Title:** Mixtral of Experts
|
||||
- **Authors:** Albert Q. Jiang, Alexandre Sablayrolles, Antoine Roux, et al.
|
||||
- **Published Date:** 2024-01-08
|
||||
- **URL:** http://arxiv.org/abs/2401.04088v1
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [together_ai](https://github.com/langchain-ai/langchain/blob/master/cookbook/together_ai.ipynb)
|
||||
|
||||
**Abstract:** We introduce Mixtral 8x7B, a Sparse Mixture of Experts (SMoE) language model.
|
||||
Mixtral has the same architecture as Mistral 7B, with the difference that each
|
||||
layer is composed of 8 feedforward blocks (i.e. experts). For every token, at
|
||||
each layer, a router network selects two experts to process the current state
|
||||
and combine their outputs. Even though each token only sees two experts, the
|
||||
selected experts can be different at each timestep. As a result, each token has
|
||||
access to 47B parameters, but only uses 13B active parameters during inference.
|
||||
Mixtral was trained with a context size of 32k tokens and it outperforms or
|
||||
matches Llama 2 70B and GPT-3.5 across all evaluated benchmarks. In particular,
|
||||
Mixtral vastly outperforms Llama 2 70B on mathematics, code generation, and
|
||||
multilingual benchmarks. We also provide a model fine-tuned to follow
|
||||
instructions, Mixtral 8x7B - Instruct, that surpasses GPT-3.5 Turbo,
|
||||
Claude-2.1, Gemini Pro, and Llama 2 70B - chat model on human benchmarks. Both
|
||||
the base and instruct models are released under the Apache 2.0 license.
|
||||
|
||||
## Dense X Retrieval: What Retrieval Granularity Should We Use?
|
||||
|
||||
- **arXiv id:** 2312.06648v2
|
||||
@ -91,6 +209,39 @@ average improvement of +7.9 in EM score given entirely noisy retrieved
|
||||
documents and +10.5 in rejection rates for real-time questions that fall
|
||||
outside the pre-training knowledge scope.
|
||||
|
||||
## Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection
|
||||
|
||||
- **arXiv id:** 2310.11511v1
|
||||
- **Title:** Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection
|
||||
- **Authors:** Akari Asai, Zeqiu Wu, Yizhong Wang, et al.
|
||||
- **Published Date:** 2023-10-17
|
||||
- **URL:** http://arxiv.org/abs/2310.11511v1
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [langgraph_self_rag](https://github.com/langchain-ai/langchain/blob/master/cookbook/langgraph_self_rag.ipynb)
|
||||
|
||||
**Abstract:** Despite their remarkable capabilities, large language models (LLMs) often
|
||||
produce responses containing factual inaccuracies due to their sole reliance on
|
||||
the parametric knowledge they encapsulate. Retrieval-Augmented Generation
|
||||
(RAG), an ad hoc approach that augments LMs with retrieval of relevant
|
||||
knowledge, decreases such issues. However, indiscriminately retrieving and
|
||||
incorporating a fixed number of retrieved passages, regardless of whether
|
||||
retrieval is necessary, or passages are relevant, diminishes LM versatility or
|
||||
can lead to unhelpful response generation. We introduce a new framework called
|
||||
Self-Reflective Retrieval-Augmented Generation (Self-RAG) that enhances an LM's
|
||||
quality and factuality through retrieval and self-reflection. Our framework
|
||||
trains a single arbitrary LM that adaptively retrieves passages on-demand, and
|
||||
generates and reflects on retrieved passages and its own generations using
|
||||
special tokens, called reflection tokens. Generating reflection tokens makes
|
||||
the LM controllable during the inference phase, enabling it to tailor its
|
||||
behavior to diverse task requirements. Experiments show that Self-RAG (7B and
|
||||
13B parameters) significantly outperforms state-of-the-art LLMs and
|
||||
retrieval-augmented models on a diverse set of tasks. Specifically, Self-RAG
|
||||
outperforms ChatGPT and retrieval-augmented Llama2-chat on Open-domain QA,
|
||||
reasoning and fact verification tasks, and it shows significant gains in
|
||||
improving factuality and citation accuracy for long-form generations relative
|
||||
to these models.
|
||||
|
||||
## Take a Step Back: Evoking Reasoning via Abstraction in Large Language Models
|
||||
|
||||
- **arXiv id:** 2310.06117v2
|
||||
@ -101,6 +252,7 @@ outside the pre-training knowledge scope.
|
||||
- **LangChain:**
|
||||
|
||||
- **Template:** [stepback-qa-prompting](https://python.langchain.com/docs/templates/stepback-qa-prompting)
|
||||
- **Cookbook:** [stepback-qa](https://github.com/langchain-ai/langchain/blob/master/cookbook/stepback-qa.ipynb)
|
||||
|
||||
**Abstract:** We present Step-Back Prompting, a simple prompting technique that enables
|
||||
LLMs to do abstractions to derive high-level concepts and first principles from
|
||||
@ -113,6 +265,27 @@ including STEM, Knowledge QA, and Multi-Hop Reasoning. For instance, Step-Back
|
||||
Prompting improves PaLM-2L performance on MMLU (Physics and Chemistry) by 7%
|
||||
and 11% respectively, TimeQA by 27%, and MuSiQue by 7%.
|
||||
|
||||
## Llama 2: Open Foundation and Fine-Tuned Chat Models
|
||||
|
||||
- **arXiv id:** 2307.09288v2
|
||||
- **Title:** Llama 2: Open Foundation and Fine-Tuned Chat Models
|
||||
- **Authors:** Hugo Touvron, Louis Martin, Kevin Stone, et al.
|
||||
- **Published Date:** 2023-07-18
|
||||
- **URL:** http://arxiv.org/abs/2307.09288v2
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [Semi_Structured_RAG](https://github.com/langchain-ai/langchain/blob/master/cookbook/Semi_Structured_RAG.ipynb)
|
||||
|
||||
**Abstract:** In this work, we develop and release Llama 2, a collection of pretrained and
|
||||
fine-tuned large language models (LLMs) ranging in scale from 7 billion to 70
|
||||
billion parameters. Our fine-tuned LLMs, called Llama 2-Chat, are optimized for
|
||||
dialogue use cases. Our models outperform open-source chat models on most
|
||||
benchmarks we tested, and based on our human evaluations for helpfulness and
|
||||
safety, may be a suitable substitute for closed-source models. We provide a
|
||||
detailed description of our approach to fine-tuning and safety improvements of
|
||||
Llama 2-Chat in order to enable the community to build on our work and
|
||||
contribute to the responsible development of LLMs.
|
||||
|
||||
## Query Rewriting for Retrieval-Augmented Large Language Models
|
||||
|
||||
- **arXiv id:** 2305.14283v3
|
||||
@ -123,6 +296,7 @@ and 11% respectively, TimeQA by 27%, and MuSiQue by 7%.
|
||||
- **LangChain:**
|
||||
|
||||
- **Template:** [rewrite-retrieve-read](https://python.langchain.com/docs/templates/rewrite-retrieve-read)
|
||||
- **Cookbook:** [rewrite](https://github.com/langchain-ai/langchain/blob/master/cookbook/rewrite.ipynb)
|
||||
|
||||
**Abstract:** Large Language Models (LLMs) play powerful, black-box readers in the
|
||||
retrieve-then-read pipeline, making remarkable progress in knowledge-intensive
|
||||
@ -152,6 +326,7 @@ for retrieval-augmented LLM.
|
||||
- **LangChain:**
|
||||
|
||||
- **API Reference:** [langchain_experimental.tot](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.tot)
|
||||
- **Cookbook:** [tree_of_thought](https://github.com/langchain-ai/langchain/blob/master/cookbook/tree_of_thought.ipynb)
|
||||
|
||||
**Abstract:** In this paper, we introduce the Tree-of-Thought (ToT) framework, a novel
|
||||
approach aimed at improving the problem-solving capabilities of auto-regressive
|
||||
@ -171,6 +346,132 @@ significantly increase the success rate of Sudoku puzzle solving. Our
|
||||
implementation of the ToT-based Sudoku solver is available on GitHub:
|
||||
\url{https://github.com/jieyilong/tree-of-thought-puzzle-solver}.
|
||||
|
||||
## Plan-and-Solve Prompting: Improving Zero-Shot Chain-of-Thought Reasoning by Large Language Models
|
||||
|
||||
- **arXiv id:** 2305.04091v3
|
||||
- **Title:** Plan-and-Solve Prompting: Improving Zero-Shot Chain-of-Thought Reasoning by Large Language Models
|
||||
- **Authors:** Lei Wang, Wanyu Xu, Yihuai Lan, et al.
|
||||
- **Published Date:** 2023-05-06
|
||||
- **URL:** http://arxiv.org/abs/2305.04091v3
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [plan_and_execute_agent](https://github.com/langchain-ai/langchain/blob/master/cookbook/plan_and_execute_agent.ipynb)
|
||||
|
||||
**Abstract:** Large language models (LLMs) have recently been shown to deliver impressive
|
||||
performance in various NLP tasks. To tackle multi-step reasoning tasks,
|
||||
few-shot chain-of-thought (CoT) prompting includes a few manually crafted
|
||||
step-by-step reasoning demonstrations which enable LLMs to explicitly generate
|
||||
reasoning steps and improve their reasoning task accuracy. To eliminate the
|
||||
manual effort, Zero-shot-CoT concatenates the target problem statement with
|
||||
"Let's think step by step" as an input prompt to LLMs. Despite the success of
|
||||
Zero-shot-CoT, it still suffers from three pitfalls: calculation errors,
|
||||
missing-step errors, and semantic misunderstanding errors. To address the
|
||||
missing-step errors, we propose Plan-and-Solve (PS) Prompting. It consists of
|
||||
two components: first, devising a plan to divide the entire task into smaller
|
||||
subtasks, and then carrying out the subtasks according to the plan. To address
|
||||
the calculation errors and improve the quality of generated reasoning steps, we
|
||||
extend PS prompting with more detailed instructions and derive PS+ prompting.
|
||||
We evaluate our proposed prompting strategy on ten datasets across three
|
||||
reasoning problems. The experimental results over GPT-3 show that our proposed
|
||||
zero-shot prompting consistently outperforms Zero-shot-CoT across all datasets
|
||||
by a large margin, is comparable to or exceeds Zero-shot-Program-of-Thought
|
||||
Prompting, and has comparable performance with 8-shot CoT prompting on the math
|
||||
reasoning problem. The code can be found at
|
||||
https://github.com/AGI-Edgerunners/Plan-and-Solve-Prompting.
|
||||
|
||||
## Visual Instruction Tuning
|
||||
|
||||
- **arXiv id:** 2304.08485v2
|
||||
- **Title:** Visual Instruction Tuning
|
||||
- **Authors:** Haotian Liu, Chunyuan Li, Qingyang Wu, et al.
|
||||
- **Published Date:** 2023-04-17
|
||||
- **URL:** http://arxiv.org/abs/2304.08485v2
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [Semi_structured_and_multi_modal_RAG](https://github.com/langchain-ai/langchain/blob/master/cookbook/Semi_structured_and_multi_modal_RAG.ipynb), [Semi_structured_multi_modal_RAG_LLaMA2](https://github.com/langchain-ai/langchain/blob/master/cookbook/Semi_structured_multi_modal_RAG_LLaMA2.ipynb)
|
||||
|
||||
**Abstract:** Instruction tuning large language models (LLMs) using machine-generated
|
||||
instruction-following data has improved zero-shot capabilities on new tasks,
|
||||
but the idea is less explored in the multimodal field. In this paper, we
|
||||
present the first attempt to use language-only GPT-4 to generate multimodal
|
||||
language-image instruction-following data. By instruction tuning on such
|
||||
generated data, we introduce LLaVA: Large Language and Vision Assistant, an
|
||||
end-to-end trained large multimodal model that connects a vision encoder and
|
||||
LLM for general-purpose visual and language understanding.Our early experiments
|
||||
show that LLaVA demonstrates impressive multimodel chat abilities, sometimes
|
||||
exhibiting the behaviors of multimodal GPT-4 on unseen images/instructions, and
|
||||
yields a 85.1% relative score compared with GPT-4 on a synthetic multimodal
|
||||
instruction-following dataset. When fine-tuned on Science QA, the synergy of
|
||||
LLaVA and GPT-4 achieves a new state-of-the-art accuracy of 92.53%. We make
|
||||
GPT-4 generated visual instruction tuning data, our model and code base
|
||||
publicly available.
|
||||
|
||||
## Generative Agents: Interactive Simulacra of Human Behavior
|
||||
|
||||
- **arXiv id:** 2304.03442v2
|
||||
- **Title:** Generative Agents: Interactive Simulacra of Human Behavior
|
||||
- **Authors:** Joon Sung Park, Joseph C. O'Brien, Carrie J. Cai, et al.
|
||||
- **Published Date:** 2023-04-07
|
||||
- **URL:** http://arxiv.org/abs/2304.03442v2
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [multiagent_bidding](https://github.com/langchain-ai/langchain/blob/master/cookbook/multiagent_bidding.ipynb), [generative_agents_interactive_simulacra_of_human_behavior](https://github.com/langchain-ai/langchain/blob/master/cookbook/generative_agents_interactive_simulacra_of_human_behavior.ipynb)
|
||||
|
||||
**Abstract:** Believable proxies of human behavior can empower interactive applications
|
||||
ranging from immersive environments to rehearsal spaces for interpersonal
|
||||
communication to prototyping tools. In this paper, we introduce generative
|
||||
agents--computational software agents that simulate believable human behavior.
|
||||
Generative agents wake up, cook breakfast, and head to work; artists paint,
|
||||
while authors write; they form opinions, notice each other, and initiate
|
||||
conversations; they remember and reflect on days past as they plan the next
|
||||
day. To enable generative agents, we describe an architecture that extends a
|
||||
large language model to store a complete record of the agent's experiences
|
||||
using natural language, synthesize those memories over time into higher-level
|
||||
reflections, and retrieve them dynamically to plan behavior. We instantiate
|
||||
generative agents to populate an interactive sandbox environment inspired by
|
||||
The Sims, where end users can interact with a small town of twenty five agents
|
||||
using natural language. In an evaluation, these generative agents produce
|
||||
believable individual and emergent social behaviors: for example, starting with
|
||||
only a single user-specified notion that one agent wants to throw a Valentine's
|
||||
Day party, the agents autonomously spread invitations to the party over the
|
||||
next two days, make new acquaintances, ask each other out on dates to the
|
||||
party, and coordinate to show up for the party together at the right time. We
|
||||
demonstrate through ablation that the components of our agent
|
||||
architecture--observation, planning, and reflection--each contribute critically
|
||||
to the believability of agent behavior. By fusing large language models with
|
||||
computational, interactive agents, this work introduces architectural and
|
||||
interaction patterns for enabling believable simulations of human behavior.
|
||||
|
||||
## CAMEL: Communicative Agents for "Mind" Exploration of Large Language Model Society
|
||||
|
||||
- **arXiv id:** 2303.17760v2
|
||||
- **Title:** CAMEL: Communicative Agents for "Mind" Exploration of Large Language Model Society
|
||||
- **Authors:** Guohao Li, Hasan Abed Al Kader Hammoud, Hani Itani, et al.
|
||||
- **Published Date:** 2023-03-31
|
||||
- **URL:** http://arxiv.org/abs/2303.17760v2
|
||||
- **LangChain:**
|
||||
|
||||
- **Cookbook:** [camel_role_playing](https://github.com/langchain-ai/langchain/blob/master/cookbook/camel_role_playing.ipynb)
|
||||
|
||||
**Abstract:** The rapid advancement of chat-based language models has led to remarkable
|
||||
progress in complex task-solving. However, their success heavily relies on
|
||||
human input to guide the conversation, which can be challenging and
|
||||
time-consuming. This paper explores the potential of building scalable
|
||||
techniques to facilitate autonomous cooperation among communicative agents, and
|
||||
provides insight into their "cognitive" processes. To address the challenges of
|
||||
achieving autonomous cooperation, we propose a novel communicative agent
|
||||
framework named role-playing. Our approach involves using inception prompting
|
||||
to guide chat agents toward task completion while maintaining consistency with
|
||||
human intentions. We showcase how role-playing can be used to generate
|
||||
conversational data for studying the behaviors and capabilities of a society of
|
||||
agents, providing a valuable resource for investigating conversational language
|
||||
models. In particular, we conduct comprehensive studies on
|
||||
instruction-following cooperation in multi-agent settings. Our contributions
|
||||
include introducing a novel communicative agent framework, offering a scalable
|
||||
approach for studying the cooperative behaviors and capabilities of multi-agent
|
||||
systems, and open-sourcing our library to support research on communicative
|
||||
agents and beyond: https://github.com/camel-ai/camel.
|
||||
|
||||
## HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in Hugging Face
|
||||
|
||||
- **arXiv id:** 2303.17580v4
|
||||
@ -181,6 +482,7 @@ implementation of the ToT-based Sudoku solver is available on GitHub:
|
||||
- **LangChain:**
|
||||
|
||||
- **API Reference:** [langchain_experimental.autonomous_agents](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.autonomous_agents)
|
||||
- **Cookbook:** [hugginggpt](https://github.com/langchain-ai/langchain/blob/master/cookbook/hugginggpt.ipynb)
|
||||
|
||||
**Abstract:** Solving complicated AI tasks with different domains and modalities is a key
|
||||
step toward artificial general intelligence. While there are numerous AI models
|
||||
@ -235,7 +537,7 @@ more than 1/1,000th the compute of GPT-4.
|
||||
- **URL:** http://arxiv.org/abs/2301.10226v4
|
||||
- **LangChain:**
|
||||
|
||||
- **API Reference:** [langchain_community.llms...OCIModelDeploymentTGI](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI.html#langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI), [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
- **API Reference:** [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_huggingface.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community.llms...OCIModelDeploymentTGI](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI.html#langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI)
|
||||
|
||||
**Abstract:** Potential harms of large language models can be mitigated by watermarking
|
||||
model output, i.e., embedding signals into generated text that are invisible to
|
||||
@ -262,6 +564,7 @@ family, and discuss robustness and security.
|
||||
|
||||
- **API Reference:** [langchain.chains...HypotheticalDocumentEmbedder](https://api.python.langchain.com/en/latest/chains/langchain.chains.hyde.base.HypotheticalDocumentEmbedder.html#langchain.chains.hyde.base.HypotheticalDocumentEmbedder)
|
||||
- **Template:** [hyde](https://python.langchain.com/docs/templates/hyde)
|
||||
- **Cookbook:** [hypothetical_document_embeddings](https://github.com/langchain-ai/langchain/blob/master/cookbook/hypothetical_document_embeddings.ipynb)
|
||||
|
||||
**Abstract:** While dense retrieval has been shown effective and efficient across tasks and
|
||||
languages, it remains difficult to create effective fully zero-shot dense
|
||||
@ -351,7 +654,8 @@ performance across three real-world tasks on multiple LLMs.
|
||||
- **URL:** http://arxiv.org/abs/2211.10435v2
|
||||
- **LangChain:**
|
||||
|
||||
- **API Reference:** [langchain_experimental.pal_chain...PALChain](https://api.python.langchain.com/en/latest/pal_chain/langchain_experimental.pal_chain.base.PALChain.html#langchain_experimental.pal_chain.base.PALChain), [langchain_experimental.pal_chain](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.pal_chain)
|
||||
- **API Reference:** [langchain_experimental.pal_chain](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.pal_chain), [langchain_experimental.pal_chain...PALChain](https://api.python.langchain.com/en/latest/pal_chain/langchain_experimental.pal_chain.base.PALChain.html#langchain_experimental.pal_chain.base.PALChain)
|
||||
- **Cookbook:** [program_aided_language_model](https://github.com/langchain-ai/langchain/blob/master/cookbook/program_aided_language_model.ipynb)
|
||||
|
||||
**Abstract:** Large language models (LLMs) have recently demonstrated an impressive ability
|
||||
to perform arithmetic and symbolic reasoning tasks, when provided with a few
|
||||
@ -442,7 +746,7 @@ encoders, mine bitexts, and validate the bitexts by training NMT systems.
|
||||
- **URL:** http://arxiv.org/abs/2204.00498v1
|
||||
- **LangChain:**
|
||||
|
||||
- **API Reference:** [langchain_community.utilities...SQLDatabase](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.sql_database.SQLDatabase.html#langchain_community.utilities.sql_database.SQLDatabase), [langchain_community.utilities...SparkSQL](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.spark_sql.SparkSQL.html#langchain_community.utilities.spark_sql.SparkSQL)
|
||||
- **API Reference:** [langchain_community.utilities...SparkSQL](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.spark_sql.SparkSQL.html#langchain_community.utilities.spark_sql.SparkSQL), [langchain_community.utilities...SQLDatabase](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.sql_database.SQLDatabase.html#langchain_community.utilities.sql_database.SQLDatabase)
|
||||
|
||||
**Abstract:** We perform an empirical evaluation of Text-to-SQL capabilities of the Codex
|
||||
language model. We find that, without any finetuning, Codex is a strong
|
||||
@ -461,7 +765,7 @@ few-shot examples.
|
||||
- **URL:** http://arxiv.org/abs/2202.00666v5
|
||||
- **LangChain:**
|
||||
|
||||
- **API Reference:** [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
- **API Reference:** [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_huggingface.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
|
||||
**Abstract:** Today's probabilistic language generators fall short when it comes to
|
||||
producing coherent and fluent text despite the fact that the underlying models
|
||||
@ -525,7 +829,7 @@ https://github.com/OpenAI/CLIP.
|
||||
- **URL:** http://arxiv.org/abs/1909.05858v2
|
||||
- **LangChain:**
|
||||
|
||||
- **API Reference:** [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
- **API Reference:** [langchain_community.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community.llms...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_huggingface.llms...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint)
|
||||
|
||||
**Abstract:** Large-scale language models show promising text generation capabilities, but
|
||||
users cannot easily control particular aspects of the generated text. We
|
||||
|
||||
@ -7,7 +7,7 @@ import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Set
|
||||
from typing import Any, Dict
|
||||
|
||||
from pydantic.v1 import BaseModel, root_validator
|
||||
|
||||
@ -17,6 +17,7 @@ _ROOT_DIR = Path(os.path.abspath(__file__)).parents[2]
|
||||
DOCS_DIR = _ROOT_DIR / "docs" / "docs"
|
||||
CODE_DIR = _ROOT_DIR / "libs"
|
||||
TEMPLATES_DIR = _ROOT_DIR / "templates"
|
||||
COOKBOOKS_DIR = _ROOT_DIR / "cookbook"
|
||||
ARXIV_ID_PATTERN = r"https://arxiv\.org/(abs|pdf)/(\d+\.\d+)"
|
||||
LANGCHAIN_PYTHON_URL = "python.langchain.com"
|
||||
|
||||
@ -29,6 +30,7 @@ class ArxivPaper:
|
||||
referencing_doc2url: dict[str, str]
|
||||
referencing_api_ref2url: dict[str, str]
|
||||
referencing_template2url: dict[str, str]
|
||||
referencing_cookbook2url: dict[str, str]
|
||||
title: str
|
||||
authors: list[str]
|
||||
abstract: str
|
||||
@ -50,7 +52,6 @@ def search_documentation_for_arxiv_references(docs_dir: Path) -> dict[str, set[s
|
||||
arxiv_url_pattern = re.compile(ARXIV_ID_PATTERN)
|
||||
exclude_strings = {"file_path", "metadata", "link", "loader", "PyPDFLoader"}
|
||||
|
||||
# loop all the files (ipynb, mdx, md) in the docs folder
|
||||
files = (
|
||||
p.resolve()
|
||||
for p in Path(docs_dir).glob("**/*")
|
||||
@ -76,39 +77,6 @@ def search_documentation_for_arxiv_references(docs_dir: Path) -> dict[str, set[s
|
||||
return arxiv_id2file_names
|
||||
|
||||
|
||||
def convert_module_name_and_members_to_urls(
|
||||
arxiv_id2module_name_and_members: dict[str, set[str]],
|
||||
) -> dict[str, set[str]]:
|
||||
arxiv_id2urls = {}
|
||||
for arxiv_id, module_name_and_members in arxiv_id2module_name_and_members.items():
|
||||
urls = set()
|
||||
for module_name_and_member in module_name_and_members:
|
||||
module_name, type_and_member = module_name_and_member.split(":")
|
||||
if "$" in type_and_member:
|
||||
type, member = type_and_member.split("$")
|
||||
else:
|
||||
type = type_and_member
|
||||
member = ""
|
||||
_namespace_parts = module_name.split(".")
|
||||
if type == "module":
|
||||
first_namespace_part = _namespace_parts[0]
|
||||
if first_namespace_part.startswith("langchain_"):
|
||||
first_namespace_part = first_namespace_part.replace(
|
||||
"langchain_", ""
|
||||
)
|
||||
url = f"{first_namespace_part}_api_reference.html#module-{module_name}"
|
||||
elif type in ["class", "function"]:
|
||||
second_namespace_part = _namespace_parts[1]
|
||||
url = f"{second_namespace_part}/{module_name}.{member}.html#{module_name}.{member}"
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown type: {type} in the {module_name_and_member}."
|
||||
)
|
||||
urls.add(url)
|
||||
arxiv_id2urls[arxiv_id] = urls
|
||||
return arxiv_id2urls
|
||||
|
||||
|
||||
def search_code_for_arxiv_references(code_dir: Path) -> dict[str, set[str]]:
|
||||
"""Search the code for arXiv references.
|
||||
|
||||
@ -220,7 +188,6 @@ def search_code_for_arxiv_references(code_dir: Path) -> dict[str, set[str]]:
|
||||
|
||||
def search_templates_for_arxiv_references(templates_dir: Path) -> dict[str, set[str]]:
|
||||
arxiv_url_pattern = re.compile(ARXIV_ID_PATTERN)
|
||||
# exclude_strings = {"file_path", "metadata", "link", "loader", "PyPDFLoader"}
|
||||
|
||||
# loop all the Readme.md files since they are parsed into LangChain documentation
|
||||
# exclude the Readme.md in the root folder
|
||||
@ -234,8 +201,6 @@ def search_templates_for_arxiv_references(templates_dir: Path) -> dict[str, set[
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
# if any(exclude_string in line for exclude_string in exclude_strings):
|
||||
# continue
|
||||
matches = arxiv_url_pattern.search(line)
|
||||
if matches:
|
||||
arxiv_id = matches.group(2)
|
||||
@ -247,6 +212,58 @@ def search_templates_for_arxiv_references(templates_dir: Path) -> dict[str, set[
|
||||
return arxiv_id2template_names
|
||||
|
||||
|
||||
def search_cookbooks_for_arxiv_references(cookbooks_dir: Path) -> dict[str, set[str]]:
|
||||
arxiv_url_pattern = re.compile(ARXIV_ID_PATTERN)
|
||||
files = (p.resolve() for p in Path(cookbooks_dir).glob("**/*.ipynb"))
|
||||
arxiv_id2cookbook_names: dict[str, set[str]] = {}
|
||||
for file in files:
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
matches = arxiv_url_pattern.search(line)
|
||||
if matches:
|
||||
arxiv_id = matches.group(2)
|
||||
cookbook_name = file.stem
|
||||
if arxiv_id not in arxiv_id2cookbook_names:
|
||||
arxiv_id2cookbook_names[arxiv_id] = {cookbook_name}
|
||||
else:
|
||||
arxiv_id2cookbook_names[arxiv_id].add(cookbook_name)
|
||||
return arxiv_id2cookbook_names
|
||||
|
||||
|
||||
def convert_module_name_and_members_to_urls(
|
||||
arxiv_id2module_name_and_members: dict[str, set[str]],
|
||||
) -> dict[str, set[str]]:
|
||||
arxiv_id2urls = {}
|
||||
for arxiv_id, module_name_and_members in arxiv_id2module_name_and_members.items():
|
||||
urls = set()
|
||||
for module_name_and_member in module_name_and_members:
|
||||
module_name, type_and_member = module_name_and_member.split(":")
|
||||
if "$" in type_and_member:
|
||||
type_, member = type_and_member.split("$")
|
||||
else:
|
||||
type_ = type_and_member
|
||||
member = ""
|
||||
_namespace_parts = module_name.split(".")
|
||||
if type_ == "module":
|
||||
first_namespace_part = _namespace_parts[0]
|
||||
if first_namespace_part.startswith("langchain_"):
|
||||
first_namespace_part = first_namespace_part.replace(
|
||||
"langchain_", ""
|
||||
)
|
||||
url = f"{first_namespace_part}_api_reference.html#module-{module_name}"
|
||||
elif type_ in ["class", "function"]:
|
||||
second_namespace_part = _namespace_parts[1]
|
||||
url = f"{second_namespace_part}/{module_name}.{member}.html#{module_name}.{member}"
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown type: {type_} in the {module_name_and_member}."
|
||||
)
|
||||
urls.add(url)
|
||||
arxiv_id2urls[arxiv_id] = urls
|
||||
return arxiv_id2urls
|
||||
|
||||
|
||||
def _get_doc_path(file_parts: tuple[str, ...], file_extension) -> str:
|
||||
"""Get the relative path to the documentation page
|
||||
from the absolute path of the file.
|
||||
@ -285,60 +302,6 @@ def _get_module_name(file_parts: tuple[str, ...]) -> str:
|
||||
return ".".join(ns_parts)
|
||||
|
||||
|
||||
def compound_urls(
|
||||
arxiv_id2file_names: dict[str, set[str]],
|
||||
arxiv_id2code_urls: dict[str, set[str]],
|
||||
arxiv_id2templates: dict[str, set[str]],
|
||||
) -> dict[str, dict[str, set[str]]]:
|
||||
# format urls and verify that the urls are correct
|
||||
arxiv_id2file_names_new = {}
|
||||
for arxiv_id, file_names in arxiv_id2file_names.items():
|
||||
key2urls = {
|
||||
key: _format_doc_url(key)
|
||||
for key in file_names
|
||||
if _is_url_ok(_format_doc_url(key))
|
||||
}
|
||||
if key2urls:
|
||||
arxiv_id2file_names_new[arxiv_id] = key2urls
|
||||
|
||||
arxiv_id2code_urls_new = {}
|
||||
for arxiv_id, code_urls in arxiv_id2code_urls.items():
|
||||
key2urls = {
|
||||
key: _format_api_ref_url(key)
|
||||
for key in code_urls
|
||||
if _is_url_ok(_format_api_ref_url(key))
|
||||
}
|
||||
if key2urls:
|
||||
arxiv_id2code_urls_new[arxiv_id] = key2urls
|
||||
|
||||
arxiv_id2templates_new = {}
|
||||
for arxiv_id, templates in arxiv_id2templates.items():
|
||||
key2urls = {
|
||||
key: _format_template_url(key)
|
||||
for key in templates
|
||||
if _is_url_ok(_format_template_url(key))
|
||||
}
|
||||
if key2urls:
|
||||
arxiv_id2templates_new[arxiv_id] = key2urls
|
||||
|
||||
arxiv_id2type2key2urls = dict.fromkeys(
|
||||
arxiv_id2file_names_new | arxiv_id2code_urls_new | arxiv_id2templates_new
|
||||
)
|
||||
arxiv_id2type2key2urls = {k: {} for k in arxiv_id2type2key2urls}
|
||||
for arxiv_id, key2urls in arxiv_id2file_names_new.items():
|
||||
arxiv_id2type2key2urls[arxiv_id]["docs"] = key2urls
|
||||
for arxiv_id, key2urls in arxiv_id2code_urls_new.items():
|
||||
arxiv_id2type2key2urls[arxiv_id]["apis"] = key2urls
|
||||
for arxiv_id, key2urls in arxiv_id2templates_new.items():
|
||||
arxiv_id2type2key2urls[arxiv_id]["templates"] = key2urls
|
||||
|
||||
# reverse sort by the arxiv_id (the newest papers first)
|
||||
ret = dict(
|
||||
sorted(arxiv_id2type2key2urls.items(), key=lambda item: item[0], reverse=True)
|
||||
)
|
||||
return ret
|
||||
|
||||
|
||||
def _is_url_ok(url: str) -> bool:
|
||||
"""Check if the url page is open without error."""
|
||||
import requests
|
||||
@ -424,6 +387,9 @@ class ArxivAPIWrapper(BaseModel):
|
||||
referencing_template2url=type2key2urls["templates"]
|
||||
if "templates" in type2key2urls
|
||||
else {},
|
||||
referencing_cookbook2url=type2key2urls["cookbooks"]
|
||||
if "cookbooks" in type2key2urls
|
||||
else {},
|
||||
)
|
||||
for result, type2key2urls in zip(results, arxiv_id2type2key2urls.values())
|
||||
]
|
||||
@ -443,6 +409,10 @@ def _format_template_url(template_name: str) -> str:
|
||||
return f"https://{LANGCHAIN_PYTHON_URL}/docs/templates/{template_name}"
|
||||
|
||||
|
||||
def _format_cookbook_url(cookbook_name: str) -> str:
|
||||
return f"https://github.com/langchain-ai/langchain/blob/master/cookbook/{cookbook_name}.ipynb"
|
||||
|
||||
|
||||
def _compact_module_full_name(doc_path: str) -> str:
|
||||
# agents/langchain_core.agents.AgentAction.html#langchain_core.agents.AgentAction
|
||||
module = doc_path.split("#")[1].replace("module-", "")
|
||||
@ -454,9 +424,79 @@ def _compact_module_full_name(doc_path: str) -> str:
|
||||
return module
|
||||
|
||||
|
||||
def compound_urls(
|
||||
arxiv_id2file_names: dict[str, set[str]],
|
||||
arxiv_id2code_urls: dict[str, set[str]],
|
||||
arxiv_id2templates: dict[str, set[str]],
|
||||
arxiv_id2cookbooks: dict[str, set[str]],
|
||||
) -> dict[str, dict[str, set[str]]]:
|
||||
# format urls and verify that the urls are correct
|
||||
arxiv_id2file_names_new = {}
|
||||
for arxiv_id, file_names in arxiv_id2file_names.items():
|
||||
key2urls = {
|
||||
key: _format_doc_url(key)
|
||||
for key in file_names
|
||||
if _is_url_ok(_format_doc_url(key))
|
||||
}
|
||||
if key2urls:
|
||||
arxiv_id2file_names_new[arxiv_id] = key2urls
|
||||
|
||||
arxiv_id2code_urls_new = {}
|
||||
for arxiv_id, code_urls in arxiv_id2code_urls.items():
|
||||
key2urls = {
|
||||
key: _format_api_ref_url(key)
|
||||
for key in code_urls
|
||||
if _is_url_ok(_format_api_ref_url(key))
|
||||
}
|
||||
if key2urls:
|
||||
arxiv_id2code_urls_new[arxiv_id] = key2urls
|
||||
|
||||
arxiv_id2templates_new = {}
|
||||
for arxiv_id, templates in arxiv_id2templates.items():
|
||||
key2urls = {
|
||||
key: _format_template_url(key)
|
||||
for key in templates
|
||||
if _is_url_ok(_format_template_url(key))
|
||||
}
|
||||
if key2urls:
|
||||
arxiv_id2templates_new[arxiv_id] = key2urls
|
||||
|
||||
arxiv_id2cookbooks_new = {}
|
||||
for arxiv_id, cookbooks in arxiv_id2cookbooks.items():
|
||||
key2urls = {
|
||||
key: _format_cookbook_url(key)
|
||||
for key in cookbooks
|
||||
if _is_url_ok(_format_cookbook_url(key))
|
||||
}
|
||||
if key2urls:
|
||||
arxiv_id2cookbooks_new[arxiv_id] = key2urls
|
||||
|
||||
arxiv_id2type2key2urls = dict.fromkeys(
|
||||
arxiv_id2file_names_new
|
||||
| arxiv_id2code_urls_new
|
||||
| arxiv_id2templates_new
|
||||
| arxiv_id2cookbooks_new
|
||||
)
|
||||
arxiv_id2type2key2urls = {k: {} for k in arxiv_id2type2key2urls}
|
||||
for arxiv_id, key2urls in arxiv_id2file_names_new.items():
|
||||
arxiv_id2type2key2urls[arxiv_id]["docs"] = key2urls
|
||||
for arxiv_id, key2urls in arxiv_id2code_urls_new.items():
|
||||
arxiv_id2type2key2urls[arxiv_id]["apis"] = key2urls
|
||||
for arxiv_id, key2urls in arxiv_id2templates_new.items():
|
||||
arxiv_id2type2key2urls[arxiv_id]["templates"] = key2urls
|
||||
for arxiv_id, key2urls in arxiv_id2cookbooks_new.items():
|
||||
arxiv_id2type2key2urls[arxiv_id]["cookbooks"] = key2urls
|
||||
|
||||
# reverse sort by the arxiv_id (the newest papers first)
|
||||
ret = dict(
|
||||
sorted(arxiv_id2type2key2urls.items(), key=lambda item: item[0], reverse=True)
|
||||
)
|
||||
return ret
|
||||
|
||||
|
||||
def log_results(arxiv_id2type2key2urls):
|
||||
arxiv_ids = arxiv_id2type2key2urls.keys()
|
||||
doc_number, api_number, templates_number = 0, 0, 0
|
||||
doc_number, api_number, templates_number, cookbooks_number = 0, 0, 0, 0
|
||||
for type2key2url in arxiv_id2type2key2urls.values():
|
||||
if "docs" in type2key2url:
|
||||
doc_number += len(type2key2url["docs"])
|
||||
@ -464,9 +504,11 @@ def log_results(arxiv_id2type2key2urls):
|
||||
api_number += len(type2key2url["apis"])
|
||||
if "templates" in type2key2url:
|
||||
templates_number += len(type2key2url["templates"])
|
||||
if "cookbooks" in type2key2url:
|
||||
cookbooks_number += len(type2key2url["cookbooks"])
|
||||
logger.warning(
|
||||
f"Found {len(arxiv_ids)} arXiv references in the {doc_number} docs, {api_number} API Refs,"
|
||||
f" and {templates_number} Templates."
|
||||
f" {templates_number} Templates, and {cookbooks_number} Cookbooks."
|
||||
)
|
||||
|
||||
|
||||
@ -477,7 +519,7 @@ def generate_arxiv_references_page(file_name: Path, papers: list[ArxivPaper]) ->
|
||||
|
||||
LangChain implements the latest research in the field of Natural Language Processing.
|
||||
This page contains `arXiv` papers referenced in the LangChain Documentation, API Reference,
|
||||
and Templates.
|
||||
Templates, and Cookbooks.
|
||||
|
||||
## Summary
|
||||
|
||||
@ -510,6 +552,14 @@ and Templates.
|
||||
for key, url in paper.referencing_template2url.items()
|
||||
)
|
||||
]
|
||||
if paper.referencing_cookbook2url:
|
||||
refs += [
|
||||
"`Cookbook:` "
|
||||
+ ", ".join(
|
||||
f"[{key}]({url})"
|
||||
for key, url in paper.referencing_cookbook2url.items()
|
||||
)
|
||||
]
|
||||
refs_str = ", ".join(refs)
|
||||
|
||||
title_link = f"[{paper.title}]({paper.url})"
|
||||
@ -533,8 +583,17 @@ and Templates.
|
||||
if paper.referencing_template2url
|
||||
else ""
|
||||
)
|
||||
cookbook_refs = (
|
||||
f" - **Cookbook:** {', '.join(f'[{key}]({url})' for key, url in paper.referencing_cookbook2url.items())}"
|
||||
if paper.referencing_cookbook2url
|
||||
else ""
|
||||
)
|
||||
refs = "\n".join(
|
||||
[el for el in [docs_refs, api_ref_refs, template_refs] if el]
|
||||
[
|
||||
el
|
||||
for el in [docs_refs, api_ref_refs, template_refs, cookbook_refs]
|
||||
if el
|
||||
]
|
||||
)
|
||||
f.write(f"""
|
||||
## {paper.title}
|
||||
@ -562,8 +621,9 @@ def main():
|
||||
)
|
||||
arxiv_id2file_names = search_documentation_for_arxiv_references(DOCS_DIR)
|
||||
arxiv_id2templates = search_templates_for_arxiv_references(TEMPLATES_DIR)
|
||||
arxiv_id2cookbooks = search_cookbooks_for_arxiv_references(COOKBOOKS_DIR)
|
||||
arxiv_id2type2key2urls = compound_urls(
|
||||
arxiv_id2file_names, arxiv_id2code_urls, arxiv_id2templates
|
||||
arxiv_id2file_names, arxiv_id2code_urls, arxiv_id2templates, arxiv_id2cookbooks
|
||||
)
|
||||
log_results(arxiv_id2type2key2urls)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user