CLAWRIM Wiki

This page is read only. You can view the source, but not change it. Ask your administrator if you think this is wrong.
====== How to download features using the ArcGIS REST API ======

This script requires [[https://jqlang.github.io/jq/|jq]].
<file bash download_arcgis_rest_features.sh>
#!/bin/sh

if [ $# -lt 2 ]; then
	echo "Usage: download_arcgis_rest_features.sh REST_URL output"
	exit
fi

# URL for an ArcGIS REST API
rest_url=$1

# output filename
output=$2

# prefix
prefix=$(echo $output | sed 's/\..*//')

# starting record offset
offset=0
# max record count
count=2000

url="$rest_url/query"

# required parameters
url="$url?f=geojson"
url="$url&where=1%3D1"
url="$url&outFields=*"
url="$url&resultRecordCount=$count"
url="$url&resultOffset=---OFFSET---"

# optional parameters
# &text=
# &objectIds=
# &time=
# &geometry=
# &geometryType=esriGeometryEnvelope
# &inSR=
# &spatialRel=esriSpatialRelIntersects
# &distance=
# &units=esriSRUnit_Foot
# &relationParam=
# &returnGeometry=true
# &returnTrueCurves=false
# &maxAllowableOffset=
# &geometryPrecision=
# &outSR=
# &havingClause=
# &returnIdsOnly=false
# &returnCountOnly=false
# &orderByFields=OBJECTID
# &groupByFieldsForStatistics=
# &outStatistics=
# &returnZ=false
# &returnM=false
# &gdbVersion=
# &historicMoment=
# &returnDistinctValues=false
# &returnExtentOnly=false
# &datumTransformation=
# &parameterValues=
# &rangeValues=
# &quantizationParameters=
# &featureEncoding=esriDefault

# infinite while loop; : mean true, so while : means while true, which means forever;
# you'll have to manually break out of this infinite loop
while :; do
        # generate a filename
        fn=${prefix}_$offset.geojson
        [ -f $fn ] || wget -O $fn "$(echo "$url" | sed "s/---OFFSET---/$offset/")"
	# read wget error
        err=$?
	# if there was a wget error or the filesize is zero
        if [ $err -ne 0 -o ! -s $fn ]; then
		# print the wget error
                [ $err -ne 0 ] && echo "$fn: wget error $err"
		# delete broken file
                rm -f $fn
		# try it again
                continue
        fi
	# we will only stop when we reached the last record
        grep -q "exceededTransferLimit.*true" $fn || break
	# move to the next offset
        offset=$(expr $offset + $count)
done

# merge geojson files
jq 'reduce inputs.features as $s (.; .features += $s) | del(.properties)' *.geojson > $output
</file>

Here is an example:
<code bash>
./download_arcgis_rest_features.sh https://hazards.fema.gov/arcgis/rest/services/FIRMette/NFHLREST_FIRMette/MapServer/1 firm_panels
</code>