diff --git a/docs/extras/integrations/document_loaders/arcgis.ipynb b/docs/extras/integrations/document_loaders/arcgis.ipynb index bdb3eb1f01..9420d4c4e0 100644 --- a/docs/extras/integrations/document_loaders/arcgis.ipynb +++ b/docs/extras/integrations/document_loaders/arcgis.ipynb @@ -5,7 +5,7 @@ "id": "62359e08-cf80-4210-a30c-f450000e65b9", "metadata": {}, "source": [ - "# ArcGISLoader\n", + "# ArcGIS\n", "\n", "This notebook demonstrates the use of the `langchain.document_loaders.ArcGISLoader` class.\n", "\n", @@ -39,8 +39,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.04 ms, sys: 1.63 ms, total: 5.67 ms\n", - "Wall time: 644 ms\n" + "CPU times: user 7.86 ms, sys: 0 ns, total: 7.86 ms\n", + "Wall time: 802 ms\n" ] } ], @@ -59,7 +59,179 @@ { "data": { "text/plain": [ - "dict_keys(['url', 'layer_description', 'item_description', 'layer_properties'])" + "{'accessed': '2023-08-15T04:30:41.689270+00:00Z',\n", + " 'name': 'Beach Ramps',\n", + " 'url': 'https://maps1.vcgov.org/arcgis/rest/services/Beaches/MapServer/7',\n", + " 'layer_description': '(Not Provided)',\n", + " 'item_description': '(Not Provided)',\n", + " 'layer_properties': {\n", + " \"currentVersion\": 10.81,\n", + " \"id\": 7,\n", + " \"name\": \"Beach Ramps\",\n", + " \"type\": \"Feature Layer\",\n", + " \"description\": \"\",\n", + " \"geometryType\": \"esriGeometryPoint\",\n", + " \"sourceSpatialReference\": {\n", + " \"wkid\": 2881,\n", + " \"latestWkid\": 2881\n", + " },\n", + " \"copyrightText\": \"\",\n", + " \"parentLayer\": null,\n", + " \"subLayers\": [],\n", + " \"minScale\": 750000,\n", + " \"maxScale\": 0,\n", + " \"drawingInfo\": {\n", + " \"renderer\": {\n", + " \"type\": \"simple\",\n", + " \"symbol\": {\n", + " \"type\": \"esriPMS\",\n", + " \"url\": \"9bb2e5ca499bb68aa3ee0d4e1ecc3849\",\n", + " \"imageData\": \"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAOxAAADsQBlSsOGwAAAJJJREFUOI3NkDEKg0AQRZ9kkSnSGBshR7DJqdJYeg7BMpcS0uQWQsqoCLExkcUJzGqT38zw2fcY1rEzbp7vjXz0EXC7gBxs1ABcG/8CYkCcDqwyLqsV+RlV0I/w7PzuJBArr1VB20H58Ls6h+xoFITkTwWpQJX7XSIBAnFwVj7MLAjJV/AC6G3QoAmK+74Lom04THTBEp/HCSc6AAAAAElFTkSuQmCC\",\n", + " \"contentType\": \"image/png\",\n", + " \"width\": 12,\n", + " \"height\": 12,\n", + " \"angle\": 0,\n", + " \"xoffset\": 0,\n", + " \"yoffset\": 0\n", + " },\n", + " \"label\": \"\",\n", + " \"description\": \"\"\n", + " },\n", + " \"transparency\": 0,\n", + " \"labelingInfo\": null\n", + " },\n", + " \"defaultVisibility\": true,\n", + " \"extent\": {\n", + " \"xmin\": -81.09480168806815,\n", + " \"ymin\": 28.858349245353473,\n", + " \"xmax\": -80.77512908572814,\n", + " \"ymax\": 29.41078388840041,\n", + " \"spatialReference\": {\n", + " \"wkid\": 4326,\n", + " \"latestWkid\": 4326\n", + " }\n", + " },\n", + " \"hasAttachments\": false,\n", + " \"htmlPopupType\": \"esriServerHTMLPopupTypeNone\",\n", + " \"displayField\": \"AccessName\",\n", + " \"typeIdField\": null,\n", + " \"subtypeFieldName\": null,\n", + " \"subtypeField\": null,\n", + " \"defaultSubtypeCode\": null,\n", + " \"fields\": [\n", + " {\n", + " \"name\": \"OBJECTID\",\n", + " \"type\": \"esriFieldTypeOID\",\n", + " \"alias\": \"OBJECTID\",\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"Shape\",\n", + " \"type\": \"esriFieldTypeGeometry\",\n", + " \"alias\": \"Shape\",\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"AccessName\",\n", + " \"type\": \"esriFieldTypeString\",\n", + " \"alias\": \"AccessName\",\n", + " \"length\": 40,\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"AccessID\",\n", + " \"type\": \"esriFieldTypeString\",\n", + " \"alias\": \"AccessID\",\n", + " \"length\": 50,\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"AccessType\",\n", + " \"type\": \"esriFieldTypeString\",\n", + " \"alias\": \"AccessType\",\n", + " \"length\": 25,\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"GeneralLoc\",\n", + " \"type\": \"esriFieldTypeString\",\n", + " \"alias\": \"GeneralLoc\",\n", + " \"length\": 100,\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"MilePost\",\n", + " \"type\": \"esriFieldTypeDouble\",\n", + " \"alias\": \"MilePost\",\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"City\",\n", + " \"type\": \"esriFieldTypeString\",\n", + " \"alias\": \"City\",\n", + " \"length\": 50,\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"AccessStatus\",\n", + " \"type\": \"esriFieldTypeString\",\n", + " \"alias\": \"AccessStatus\",\n", + " \"length\": 50,\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"Entry_Date_Time\",\n", + " \"type\": \"esriFieldTypeDate\",\n", + " \"alias\": \"Entry_Date_Time\",\n", + " \"length\": 8,\n", + " \"domain\": null\n", + " },\n", + " {\n", + " \"name\": \"DrivingZone\",\n", + " \"type\": \"esriFieldTypeString\",\n", + " \"alias\": \"DrivingZone\",\n", + " \"length\": 50,\n", + " \"domain\": null\n", + " }\n", + " ],\n", + " \"geometryField\": {\n", + " \"name\": \"Shape\",\n", + " \"type\": \"esriFieldTypeGeometry\",\n", + " \"alias\": \"Shape\"\n", + " },\n", + " \"indexes\": null,\n", + " \"subtypes\": [],\n", + " \"relationships\": [],\n", + " \"canModifyLayer\": true,\n", + " \"canScaleSymbols\": false,\n", + " \"hasLabels\": false,\n", + " \"capabilities\": \"Map,Query,Data\",\n", + " \"maxRecordCount\": 1000,\n", + " \"supportsStatistics\": true,\n", + " \"supportsAdvancedQueries\": true,\n", + " \"supportedQueryFormats\": \"JSON, geoJSON\",\n", + " \"isDataVersioned\": false,\n", + " \"ownershipBasedAccessControlForFeatures\": {\n", + " \"allowOthersToQuery\": true\n", + " },\n", + " \"useStandardizedQueries\": true,\n", + " \"advancedQueryCapabilities\": {\n", + " \"useStandardizedQueries\": true,\n", + " \"supportsStatistics\": true,\n", + " \"supportsHavingClause\": true,\n", + " \"supportsCountDistinct\": true,\n", + " \"supportsOrderBy\": true,\n", + " \"supportsDistinct\": true,\n", + " \"supportsPagination\": true,\n", + " \"supportsTrueCurve\": true,\n", + " \"supportsReturningQueryExtent\": true,\n", + " \"supportsQueryWithDistance\": true,\n", + " \"supportsSqlExpression\": true\n", + " },\n", + " \"supportsDatumTransformation\": true,\n", + " \"dateFieldsTimeReference\": null,\n", + " \"supportsCoordinatesQuantization\": true\n", + " }}" ] }, "execution_count": 3, @@ -68,200 +240,12 @@ } ], "source": [ - "docs[0].metadata.keys()" + "docs[0].metadata" ] }, { "cell_type": "code", "execution_count": 4, - "id": "6b6e9107-6a80-4ef7-8149-3013faa2de76", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KeysView({\n", - " \"currentVersion\": 10.81,\n", - " \"id\": 7,\n", - " \"name\": \"Beach Ramps\",\n", - " \"type\": \"Feature Layer\",\n", - " \"description\": \"\",\n", - " \"geometryType\": \"esriGeometryPoint\",\n", - " \"sourceSpatialReference\": {\n", - " \"wkid\": 2881,\n", - " \"latestWkid\": 2881\n", - " },\n", - " \"copyrightText\": \"\",\n", - " \"parentLayer\": null,\n", - " \"subLayers\": [],\n", - " \"minScale\": 750000,\n", - " \"maxScale\": 0,\n", - " \"drawingInfo\": {\n", - " \"renderer\": {\n", - " \"type\": \"simple\",\n", - " \"symbol\": {\n", - " \"type\": \"esriPMS\",\n", - " \"url\": \"9bb2e5ca499bb68aa3ee0d4e1ecc3849\",\n", - " \"imageData\": \"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAOxAAADsQBlSsOGwAAAJJJREFUOI3NkDEKg0AQRZ9kkSnSGBshR7DJqdJYeg7BMpcS0uQWQsqoCLExkcUJzGqT38zw2fcY1rEzbp7vjXz0EXC7gBxs1ABcG/8CYkCcDqwyLqsV+RlV0I/w7PzuJBArr1VB20H58Ls6h+xoFITkTwWpQJX7XSIBAnFwVj7MLAjJV/AC6G3QoAmK+74Lom04THTBEp/HCSc6AAAAAElFTkSuQmCC\",\n", - " \"contentType\": \"image/png\",\n", - " \"width\": 12,\n", - " \"height\": 12,\n", - " \"angle\": 0,\n", - " \"xoffset\": 0,\n", - " \"yoffset\": 0\n", - " },\n", - " \"label\": \"\",\n", - " \"description\": \"\"\n", - " },\n", - " \"transparency\": 0,\n", - " \"labelingInfo\": null\n", - " },\n", - " \"defaultVisibility\": true,\n", - " \"extent\": {\n", - " \"xmin\": -81.09480168806815,\n", - " \"ymin\": 28.858349245353473,\n", - " \"xmax\": -80.77512908572814,\n", - " \"ymax\": 29.41078388840041,\n", - " \"spatialReference\": {\n", - " \"wkid\": 4326,\n", - " \"latestWkid\": 4326\n", - " }\n", - " },\n", - " \"hasAttachments\": false,\n", - " \"htmlPopupType\": \"esriServerHTMLPopupTypeNone\",\n", - " \"displayField\": \"AccessName\",\n", - " \"typeIdField\": null,\n", - " \"subtypeFieldName\": null,\n", - " \"subtypeField\": null,\n", - " \"defaultSubtypeCode\": null,\n", - " \"fields\": [\n", - " {\n", - " \"name\": \"OBJECTID\",\n", - " \"type\": \"esriFieldTypeOID\",\n", - " \"alias\": \"OBJECTID\",\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"Shape\",\n", - " \"type\": \"esriFieldTypeGeometry\",\n", - " \"alias\": \"Shape\",\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"AccessName\",\n", - " \"type\": \"esriFieldTypeString\",\n", - " \"alias\": \"AccessName\",\n", - " \"length\": 40,\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"AccessID\",\n", - " \"type\": \"esriFieldTypeString\",\n", - " \"alias\": \"AccessID\",\n", - " \"length\": 50,\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"AccessType\",\n", - " \"type\": \"esriFieldTypeString\",\n", - " \"alias\": \"AccessType\",\n", - " \"length\": 25,\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"GeneralLoc\",\n", - " \"type\": \"esriFieldTypeString\",\n", - " \"alias\": \"GeneralLoc\",\n", - " \"length\": 100,\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"MilePost\",\n", - " \"type\": \"esriFieldTypeDouble\",\n", - " \"alias\": \"MilePost\",\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"City\",\n", - " \"type\": \"esriFieldTypeString\",\n", - " \"alias\": \"City\",\n", - " \"length\": 50,\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"AccessStatus\",\n", - " \"type\": \"esriFieldTypeString\",\n", - " \"alias\": \"AccessStatus\",\n", - " \"length\": 50,\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"Entry_Date_Time\",\n", - " \"type\": \"esriFieldTypeDate\",\n", - " \"alias\": \"Entry_Date_Time\",\n", - " \"length\": 8,\n", - " \"domain\": null\n", - " },\n", - " {\n", - " \"name\": \"DrivingZone\",\n", - " \"type\": \"esriFieldTypeString\",\n", - " \"alias\": \"DrivingZone\",\n", - " \"length\": 50,\n", - " \"domain\": null\n", - " }\n", - " ],\n", - " \"geometryField\": {\n", - " \"name\": \"Shape\",\n", - " \"type\": \"esriFieldTypeGeometry\",\n", - " \"alias\": \"Shape\"\n", - " },\n", - " \"indexes\": null,\n", - " \"subtypes\": [],\n", - " \"relationships\": [],\n", - " \"canModifyLayer\": true,\n", - " \"canScaleSymbols\": false,\n", - " \"hasLabels\": false,\n", - " \"capabilities\": \"Map,Query,Data\",\n", - " \"maxRecordCount\": 1000,\n", - " \"supportsStatistics\": true,\n", - " \"supportsAdvancedQueries\": true,\n", - " \"supportedQueryFormats\": \"JSON, geoJSON\",\n", - " \"isDataVersioned\": false,\n", - " \"ownershipBasedAccessControlForFeatures\": {\n", - " \"allowOthersToQuery\": true\n", - " },\n", - " \"useStandardizedQueries\": true,\n", - " \"advancedQueryCapabilities\": {\n", - " \"useStandardizedQueries\": true,\n", - " \"supportsStatistics\": true,\n", - " \"supportsHavingClause\": true,\n", - " \"supportsCountDistinct\": true,\n", - " \"supportsOrderBy\": true,\n", - " \"supportsDistinct\": true,\n", - " \"supportsPagination\": true,\n", - " \"supportsTrueCurve\": true,\n", - " \"supportsReturningQueryExtent\": true,\n", - " \"supportsQueryWithDistance\": true,\n", - " \"supportsSqlExpression\": true\n", - " },\n", - " \"supportsDatumTransformation\": true,\n", - " \"dateFieldsTimeReference\": null,\n", - " \"supportsCoordinatesQuantization\": true\n", - "})" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "docs[0].metadata['layer_properties'].keys()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, "id": "1d132b7d-5a13-4d66-98e8-785ffdf87af0", "metadata": {}, "outputs": [ @@ -269,29 +253,29 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\"OBJECTID\": 2, \"AccessName\": \"27TH AV\", \"AccessID\": \"NS-141\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3600 BLK S ATLANTIC AV\", \"MilePost\": 4.83, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"BOTH\"}\n", - "{\"OBJECTID\": 7, \"AccessName\": \"BEACHWAY AV\", \"AccessID\": \"NS-106\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1400 N ATLANTIC AV\", \"MilePost\": 1.57, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 10, \"AccessName\": \"SEABREEZE BLVD\", \"AccessID\": \"DB-051\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"500 BLK N ATLANTIC AV\", \"MilePost\": 14.24, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691394892000, \"DrivingZone\": \"BOTH\"}\n", - "{\"OBJECTID\": 13, \"AccessName\": \"GRANADA BLVD\", \"AccessID\": \"OB-030\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"20 BLK OCEAN SHORE BLVD\", \"MilePost\": 10.02, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"4X4 ONLY\", \"Entry_Date_Time\": 1691394952000, \"DrivingZone\": \"BOTH\"}\n", - "{\"OBJECTID\": 16, \"AccessName\": \"INTERNATIONAL SPEEDWAY BLVD\", \"AccessID\": \"DB-059\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"300 BLK S ATLANTIC AV\", \"MilePost\": 15.27, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691395174000, \"DrivingZone\": \"BOTH\"}\n", - "{\"OBJECTID\": 26, \"AccessName\": \"UNIVERSITY BLVD\", \"AccessID\": \"DB-048\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"900 BLK N ATLANTIC AV\", \"MilePost\": 13.74, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691394892000, \"DrivingZone\": \"BOTH\"}\n", - "{\"OBJECTID\": 36, \"AccessName\": \"BEACH ST\", \"AccessID\": \"PI-097\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"4890 BLK S ATLANTIC AV\", \"MilePost\": 25.85, \"City\": \"PONCE INLET\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"BOTH\"}\n", - "{\"OBJECTID\": 40, \"AccessName\": \"BOTEFUHR AV\", \"AccessID\": \"DBS-067\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1900 BLK S ATLANTIC AV\", \"MilePost\": 16.68, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"4X4 ONLY\", \"Entry_Date_Time\": 1691395124000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 41, \"AccessName\": \"SILVER BEACH AV\", \"AccessID\": \"DB-064\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1000 BLK S ATLANTIC AV\", \"MilePost\": 15.98, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691395174000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 50, \"AccessName\": \"3RD AV\", \"AccessID\": \"NS-118\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1200 BLK HILL ST\", \"MilePost\": 3.25, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 58, \"AccessName\": \"DUNLAWTON BLVD\", \"AccessID\": \"DBS-078\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3400 BLK S ATLANTIC AV\", \"MilePost\": 20.61, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 63, \"AccessName\": \"MILSAP RD\", \"AccessID\": \"OB-037\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"700 BLK S ATLANTIC AV\", \"MilePost\": 11.52, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"4X4 ONLY\", \"Entry_Date_Time\": 1691394952000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 68, \"AccessName\": \"EMILIA AV\", \"AccessID\": \"DBS-082\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3790 BLK S ATLANTIC AV\", \"MilePost\": 21.38, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"BOTH\"}\n", - "{\"OBJECTID\": 92, \"AccessName\": \"FLAGLER AV\", \"AccessID\": \"NS-110\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"500 BLK FLAGLER AV\", \"MilePost\": 2.57, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 94, \"AccessName\": \"CRAWFORD RD\", \"AccessID\": \"NS-108\", \"AccessType\": \"OPEN VEHICLE RAMP - PASS\", \"GeneralLoc\": \"800 BLK N ATLANTIC AV\", \"MilePost\": 2.19, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 122, \"AccessName\": \"HARTFORD AV\", \"AccessID\": \"DB-043\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1890 BLK N ATLANTIC AV\", \"MilePost\": 12.76, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"CLOSED - SEASONAL\", \"Entry_Date_Time\": 1691394832000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 125, \"AccessName\": \"WILLIAMS AV\", \"AccessID\": \"DB-042\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"2200 BLK N ATLANTIC AV\", \"MilePost\": 12.5, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"4X4 ONLY\", \"Entry_Date_Time\": 1691394952000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 134, \"AccessName\": \"CARDINAL DR\", \"AccessID\": \"OB-036\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"600 BLK S ATLANTIC AV\", \"MilePost\": 11.27, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"4X4 ONLY\", \"Entry_Date_Time\": 1691394952000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 229, \"AccessName\": \"EL PORTAL ST\", \"AccessID\": \"DBS-076\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3200 BLK S ATLANTIC AV\", \"MilePost\": 20.04, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 230, \"AccessName\": \"HARVARD DR\", \"AccessID\": \"OB-038\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"900 BLK S ATLANTIC AV\", \"MilePost\": 11.72, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"4X4 ONLY\", \"Entry_Date_Time\": 1691394952000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 232, \"AccessName\": \"VAN AV\", \"AccessID\": \"DBS-075\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3100 BLK S ATLANTIC AV\", \"MilePost\": 19.6, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"OPEN\", \"Entry_Date_Time\": 1691397348000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 233, \"AccessName\": \"ROCKEFELLER DR\", \"AccessID\": \"OB-034\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"400 BLK S ATLANTIC AV\", \"MilePost\": 10.9, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"CLOSED - SEASONAL\", \"Entry_Date_Time\": 1691394832000, \"DrivingZone\": \"YES\"}\n", - "{\"OBJECTID\": 235, \"AccessName\": \"MINERVA RD\", \"AccessID\": \"DBS-069\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"2300 BLK S ATLANTIC AV\", \"MilePost\": 17.52, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"4X4 ONLY\", \"Entry_Date_Time\": 1691395124000, \"DrivingZone\": \"YES\"}\n" + "{\"OBJECTID\": 4, \"AccessName\": \"BEACHWAY AV\", \"AccessID\": \"NS-106\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1400 N ATLANTIC AV\", \"MilePost\": 1.57, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 5, \"AccessName\": \"SEABREEZE BLVD\", \"AccessID\": \"DB-051\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"500 BLK N ATLANTIC AV\", \"MilePost\": 14.24, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"BOTH\"}\n", + "{\"OBJECTID\": 6, \"AccessName\": \"27TH AV\", \"AccessID\": \"NS-141\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3600 BLK S ATLANTIC AV\", \"MilePost\": 4.83, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"BOTH\"}\n", + "{\"OBJECTID\": 11, \"AccessName\": \"INTERNATIONAL SPEEDWAY BLVD\", \"AccessID\": \"DB-059\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"300 BLK S ATLANTIC AV\", \"MilePost\": 15.27, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"BOTH\"}\n", + "{\"OBJECTID\": 14, \"AccessName\": \"GRANADA BLVD\", \"AccessID\": \"OB-030\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"20 BLK OCEAN SHORE BLVD\", \"MilePost\": 10.02, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"BOTH\"}\n", + "{\"OBJECTID\": 27, \"AccessName\": \"UNIVERSITY BLVD\", \"AccessID\": \"DB-048\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"900 BLK N ATLANTIC AV\", \"MilePost\": 13.74, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"BOTH\"}\n", + "{\"OBJECTID\": 38, \"AccessName\": \"BEACH ST\", \"AccessID\": \"PI-097\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"4890 BLK S ATLANTIC AV\", \"MilePost\": 25.85, \"City\": \"PONCE INLET\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"BOTH\"}\n", + "{\"OBJECTID\": 42, \"AccessName\": \"BOTEFUHR AV\", \"AccessID\": \"DBS-067\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1900 BLK S ATLANTIC AV\", \"MilePost\": 16.68, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 43, \"AccessName\": \"SILVER BEACH AV\", \"AccessID\": \"DB-064\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1000 BLK S ATLANTIC AV\", \"MilePost\": 15.98, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 45, \"AccessName\": \"MILSAP RD\", \"AccessID\": \"OB-037\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"700 BLK S ATLANTIC AV\", \"MilePost\": 11.52, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 56, \"AccessName\": \"3RD AV\", \"AccessID\": \"NS-118\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1200 BLK HILL ST\", \"MilePost\": 3.25, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 64, \"AccessName\": \"DUNLAWTON BLVD\", \"AccessID\": \"DBS-078\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3400 BLK S ATLANTIC AV\", \"MilePost\": 20.61, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 69, \"AccessName\": \"EMILIA AV\", \"AccessID\": \"DBS-082\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3790 BLK S ATLANTIC AV\", \"MilePost\": 21.38, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"BOTH\"}\n", + "{\"OBJECTID\": 94, \"AccessName\": \"FLAGLER AV\", \"AccessID\": \"NS-110\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"500 BLK FLAGLER AV\", \"MilePost\": 2.57, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 96, \"AccessName\": \"CRAWFORD RD\", \"AccessID\": \"NS-108\", \"AccessType\": \"OPEN VEHICLE RAMP - PASS\", \"GeneralLoc\": \"800 BLK N ATLANTIC AV\", \"MilePost\": 2.19, \"City\": \"NEW SMYRNA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 124, \"AccessName\": \"HARTFORD AV\", \"AccessID\": \"DB-043\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"1890 BLK N ATLANTIC AV\", \"MilePost\": 12.76, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 127, \"AccessName\": \"WILLIAMS AV\", \"AccessID\": \"DB-042\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"2200 BLK N ATLANTIC AV\", \"MilePost\": 12.5, \"City\": \"DAYTONA BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 136, \"AccessName\": \"CARDINAL DR\", \"AccessID\": \"OB-036\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"600 BLK S ATLANTIC AV\", \"MilePost\": 11.27, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 229, \"AccessName\": \"EL PORTAL ST\", \"AccessID\": \"DBS-076\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3200 BLK S ATLANTIC AV\", \"MilePost\": 20.04, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 230, \"AccessName\": \"HARVARD DR\", \"AccessID\": \"OB-038\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"900 BLK S ATLANTIC AV\", \"MilePost\": 11.72, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 232, \"AccessName\": \"VAN AV\", \"AccessID\": \"DBS-075\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"3100 BLK S ATLANTIC AV\", \"MilePost\": 19.6, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 234, \"AccessName\": \"ROCKEFELLER DR\", \"AccessID\": \"OB-034\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"400 BLK S ATLANTIC AV\", \"MilePost\": 10.9, \"City\": \"ORMOND BEACH\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n", + "{\"OBJECTID\": 235, \"AccessName\": \"MINERVA RD\", \"AccessID\": \"DBS-069\", \"AccessType\": \"OPEN VEHICLE RAMP\", \"GeneralLoc\": \"2300 BLK S ATLANTIC AV\", \"MilePost\": 17.52, \"City\": \"DAYTONA BEACH SHORES\", \"AccessStatus\": \"CLOSED\", \"Entry_Date_Time\": 1692039947000, \"DrivingZone\": \"YES\"}\n" ] } ], diff --git a/libs/langchain/langchain/document_loaders/arcgis_loader.py b/libs/langchain/langchain/document_loaders/arcgis_loader.py index e1a4291b8a..8722281328 100644 --- a/libs/langchain/langchain/document_loaders/arcgis_loader.py +++ b/libs/langchain/langchain/document_loaders/arcgis_loader.py @@ -1,9 +1,11 @@ """Document Loader for ArcGIS FeatureLayers.""" + from __future__ import annotations import json import re import warnings +from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Union from langchain.docstore.document import Document @@ -25,6 +27,7 @@ class ArcGISLoader(BaseLoader): where: str = "1=1", out_fields: Optional[Union[List[str], str]] = None, return_geometry: bool = False, + return_all_records: bool = True, **kwargs: Any, ): try: @@ -64,10 +67,12 @@ class ArcGISLoader(BaseLoader): self.out_fields = ",".join(out_fields) self.return_geometry = return_geometry + self.return_all_records = return_all_records self.kwargs = kwargs def _get_layer_properties(self) -> dict: """Get the layer properties from the FeatureLayer.""" + import arcgis layer_number_pattern = re.compile(r"/\d+$") props = self.layer.properties @@ -109,7 +114,7 @@ class ArcGISLoader(BaseLoader): where=self.where, out_fields=self.out_fields, return_geometry=self.return_geometry, - return_all_records=True, + return_all_records=self.return_all_records, **self.kwargs, ) features = (feature.as_dict["attributes"] for feature in query_response) @@ -117,6 +122,8 @@ class ArcGISLoader(BaseLoader): yield Document( page_content=json.dumps(feature), metadata={ + "accessed": f"{datetime.now(timezone.utc).isoformat()}Z", + "name": self.layer_properties["layer_properties"]["name"], "url": self.url, "layer_description": self.layer_properties["layer_description"], "item_description": self.layer_properties["item_description"], diff --git a/libs/langchain/tests/unit_tests/document_loaders/test_arcgis_loader.py b/libs/langchain/tests/unit_tests/document_loaders/test_arcgis_loader.py index 645cb99e8a..a2f7d05e3e 100644 --- a/libs/langchain/tests/unit_tests/document_loaders/test_arcgis_loader.py +++ b/libs/langchain/tests/unit_tests/document_loaders/test_arcgis_loader.py @@ -24,7 +24,8 @@ def mock_feature_layer(): # type: ignore ] feature_layer.url = "https://example.com/layer_url" feature_layer.properties = { - "description": "Some HTML content" + "description": "Some HTML content", + "name": "test", } return feature_layer