fixed! now using different method for h3.

preprocess/CBN-data.ipynb

@@ -48,11 +48,11 @@
     "folder = 'Counties'\n",
     "name = 'CA_counties'\n",
     "\n",
-    "unzip(s3, folder = folder, file = '30x30_Counties.zip')\n",
+    "# unzip(s3, folder = folder, file = '30x30_Counties.zip')\n",
     "cols = process_vector(s3, folder = folder, file = f\"{name}.shp\")\n",
-    "convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
+    "# convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
     "\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols)"
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols, zoom = 8)"
    ]
   },
   {
@@ -76,11 +76,10 @@
     "\n",
     "folder = 'Climate_zones'\n",
     "name = 'climate_zones_10'\n",
-    "download(s3, folder = folder, file = 'clusters_10.tif')\n",
+    "# download(s3, folder = folder, file = 'clusters_10.tif')\n",
     "cols = process_raster(s3, folder = folder, file = 'clusters_10.tif', file_name = f\"{name}.tif\")\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols)\n",
-    "\n",
-    "\n"
+    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols,\n",
+    "          zoom = 8)\n"
    ]
   },
   {
@@ -105,10 +104,10 @@
     "folder = 'Ecoregion'\n",
     "name = 'ACE_ecoregions'\n",
     "\n",
-    "unzip(s3, folder = folder, file = '30x30_Ecoregions.zip')\n",
+    "# unzip(s3, folder = folder, file = '30x30_Ecoregions.zip')\n",
     "cols = process_vector(s3, folder = folder, file = f\"{name}.shp\")\n",
     "\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols)"
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols, zoom = 8)"
    ]
   },
   {
@@ -127,24 +126,11 @@
     "#### 13 class major habitat types **"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "df40e121-e2d4-4962-9c30-ed7e931bb705",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# download(folder = 'Habitat', file = 'CWHR13_2022.tif')\n",
-    "# cols = process_raster(s3, folder = 'Habitat', file = 'CWHR13_2022.tif')"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "de501ac3-f6fe-44f5-86c1-afba763147ae",
-   "metadata": {
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "%%time\n",
@@ -153,7 +139,6 @@
     "\n",
     "folder = 'Habitat'\n",
     "name = 'fveg22_1'\n",
-    "\n",
     "# unzip(s3, folder = folder, file = 'fveg221gdb.zip')\n",
     "\n",
     "# command = [\n",
@@ -167,24 +152,8 @@
     "cols = process_raster(s3, folder = folder, file = f\"{name}.tif\")\n",
     "# upload(folder = folder, file = f'{name}_processed.tif.aux.xml')\n",
     "\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ac178c43-f6a5-4286-a348-48bfcb1e9397",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# url = f\"s3://public-ca30x30/{folder}/{name}.parquet\"\n",
-    "\n",
-    "folder = 'Habitat'\n",
-    "name = 'fveg22_1'\n",
-    "url = f\"s3://public-ca30x30/CBN-data/{folder}/hex/{name}.parquet\"\n",
-    "\n",
-    "con.read_parquet(url).head(5).execute()\n",
-    "\n"
+    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols,\n",
+    "          zoom = 8)"
    ]
   },
   {
@@ -223,7 +192,7 @@
     "cols = process_vector(s3, folder = folder, file = f\"{name}.geojson\")\n",
     "# convert_pmtiles(con, s3, folder = folder, file = f\"{name}.geojson\")\n",
     "\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols)\n",
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols, zoom = 8)\n",
     "# gdf = gpd.read_parquet(f\"{name}.parquet\")\n"
    ]
   },
@@ -332,7 +301,7 @@
     "\n",
     "# download(s3, folder = folder, file = f\"{name}.tif\")\n",
     "cols = filter_raster(s3, folder = folder, file = f\"{name}.tif\", percentile = 80)\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols)"
+    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols, zoom = 8)"
    ]
   },
   {
@@ -359,7 +328,7 @@
     "\n",
     "download(s3, folder = folder, file = f\"{name}.tif\")\n",
     "cols = filter_raster(s3, folder = folder, file = f\"{name}.tif\", percentile = 80)\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols)"
+    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols, zoom = 8)"
    ]
   },
   {
@@ -402,7 +371,8 @@
     "name = 'rcn_wIntactBioCat_caOnly_2020-10-27'\n",
     "\n",
     "cols = process_raster(s3, folder = folder, file = f\"{name}.tif\")\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols)"
+    "convert_h3(con, s3, folder = folder, file = f\"{name}_processed.parquet\", cols = cols, \n",
+    "           zoom = 8)"
    ]
   },
   {
@@ -491,7 +461,7 @@
     "\n",
     "cols = process_vector(s3, folder = folder, file = f\"{name}.parquet\", gdf = gdf)\n",
     "convert_pmtiles(con, s3, folder =folder, file = f\"{name}.parquet\")\n",
-    "geom_to_h3(con, folder = folder, file = f\"{name}.parquet\", cols = cols)\n"
+    "geom_to_h3(con, folder = folder, file = f\"{name}.parquet\", cols = cols, zoom = 8)\n"
    ]
   },
   {
@@ -594,29 +564,29 @@
     "set_secrets(con)\n",
     "\n",
     "folder = 'NBS_agriculture/Farmland'\n",
-    "unzip(s3, folder = folder, file = 'Important_Farmland_2018.zip')\n",
+    "# unzip(s3, folder = folder, file = 'Important_Farmland_2018.zip')\n",
     "\n",
     "folder = 'NBS_agriculture/Farmland_all'\n",
     "name = 'Important_Farmland_2018'\n",
     "cols = process_vector(s3, folder = folder, file = f\"{name}.gdb\",crs = \"epsg:4326\")\n",
-    "convert_pmtiles(con, s3, folder = folder, file =f\"{name}.parquet\")\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols)\n",
+    "# convert_pmtiles(con, s3, folder = folder, file =f\"{name}.parquet\")\n",
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols, zoom = 8)\n",
     "\n",
     "# only pick a subset \n",
     "folder = 'NBS_agriculture/Farmland_all/Farmland'\n",
     "name = 'Farmland_2018'\n",
-    "gdf = gpd.read_file('Important_Farmland_2018.gdb')\n",
-    "farmland_type = ['P','S','L','U'] # prime, statewide importance, local importance, unique\n",
-    "gdf_farmland = gdf[gdf['polygon_ty'].isin(farmland_type)]\n",
-    "cols = process_vector(s3, folder = folder, file = f\"{name}.parquet\", gdf = gdf_farmland)\n",
-    "convert_pmtiles(con, s3, folder = folder, file =f\"{name}.parquet\")\n",
+    "# gdf = gpd.read_file('Important_Farmland_2018.gdb')\n",
+    "# farmland_type = ['P','S','L','U'] # prime, statewide importance, local importance, unique\n",
+    "# gdf_farmland = gdf[gdf['polygon_ty'].isin(farmland_type)]\n",
+    "# cols = process_vector(s3, folder = folder, file = f\"{name}.parquet\", gdf = gdf_farmland)\n",
+    "# convert_pmtiles(con, s3, folder = folder, file =f\"{name}.parquet\")\n",
     "\n",
     "# grazing lands \n",
     "folder = 'NBS_agriculture/Farmland_all/Lands_suitable_grazing'\n",
     "name = 'Grazing_land_2018'\n",
-    "gdf_grazing = gdf[gdf['polygon_ty'] == 'G']\n",
-    "cols = process_vector(s3, folder = folder, file = f\"{name}.parquet\", gdf = gdf_grazing)\n",
-    "convert_pmtiles(con, s3, folder = folder, file =f\"{name}.parquet\")\n"
+    "# gdf_grazing = gdf[gdf['polygon_ty'] == 'G']\n",
+    "# cols = process_vector(s3, folder = folder, file = f\"{name}.parquet\", gdf = gdf_grazing)\n",
+    "# convert_pmtiles(con, s3, folder = folder, file =f\"{name}.parquet\")\n"
    ]
   },
   {
@@ -673,9 +643,9 @@
     "# gdf['YEAR_'] = gdf['YEAR_'].astype('int64')\n",
     "# gdf = gdf[gdf['YEAR_']>=2014]\n",
     "cols = process_vector(s3, folder = folder, file = f\"{name}.parquet\", gdf = gdf)\n",
-    "convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
+    "# convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
     "\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols)"
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols, zoom = 8)"
    ]
   },
   {
@@ -759,10 +729,10 @@
     "folder = 'Progress_data_new_protection/Newly_counted_lands'\n",
     "name = 'newly_counted_lands_2024'\n",
     "\n",
-    "unzip(s3, folder = folder, file = f\"{name}.shp.zip\")\n",
-    "cols = process_vector(s3, folder = folder, file = f\"{name}.shp\")\n",
-    "convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols)\n"
+    "# unzip(s3, folder = folder, file = f\"{name}.shp.zip\")\n",
+    "cols = process_vector(s3, folder = folder, file = f\"{name}.shp\",crs = \"epsg:4326\")\n",
+    "# convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols, zoom = 8)\n"
    ]
   },
   {
@@ -790,7 +760,7 @@
     "unzip(s3, folder = folder, file = 'sb535dacgdbf2022gdb.zip')\n",
     "cols = process_vector(s3, folder = folder, file = 'SB535DACgdb_F_2022.gdb', file_name = f\"{name}.parquet\")\n",
     "convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols)\n"
+    "# convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols, zoom = 8)\n"
    ]
   },
   {
@@ -814,17 +784,17 @@
     "\n",
     "folder = 'Progress_data_new_protection/Priority_populations'\n",
     "name = 'CalEnviroScreen4'\n",
-    "unzip(s3, folder = folder, file = 'Priority Populations 4.0 Geodatabase.zip')\n",
+    "# unzip(s3, folder = folder, file = 'Priority Populations 4.0 Geodatabase.zip')\n",
     "\n",
     "gdf = (con.read_geo('Priority Populations 4.0 Combined Layer.gdb')\n",
     "       .mutate(id=ibis.row_number().over()) #making a unique id \n",
     "      ).execute().set_crs('EPSG:3857')\n",
     "\n",
-    "cols = process_vector(folder = folder, file = 'Priority Populations 4.0 Combined Layer.gdb',\n",
+    "cols = process_vector(s3, folder = folder, file = 'Priority Populations 4.0 Combined Layer.gdb',\n",
     "               file_name = f\"{name}.parquet\", gdf = gdf)\n",
     "\n",
-    "convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols)\n"
+    "# convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols, zoom = 8)\n"
    ]
   },
   {
@@ -903,13 +873,14 @@
     "set_secrets(con)\n",
     "\n",
     "# file = 'ca-30x30-base.parquet'\n",
-    "folder = \"Preprocessing\"\n",
+    "folder = \"CA_Nature/2024/Preprocessing\"\n",
     "name = 'ca-30x30-base'\n",
-    "# download(folder = folder, file = f\"{name}.parquet\")\n",
+    "download(s3, folder = folder, file = f\"{name}.parquet\")\n",
     "\n",
     "# gdf = gpd.read_parquet(f\"{name}.parquet\")\n",
-    "cols = process_vector(s3, folder = folder, file = f\"{name}.parquet\")\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols)\n"
+    "cols = process_vector(s3, folder = folder, file = f\"{name}.parquet\", crs=\"EPSG:4326\")\n",
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols = cols, \n",
+    "           zoom = 8)\n"
    ]
   },
   {
@@ -930,21 +901,21 @@
     "con = ibis.duckdb.connect('cpad',extensions = [\"spatial\", \"h3\"])\n",
     "set_secrets(con)\n",
     "\n",
-    "folder = 'cpad'\n",
+    "folder = 'CPAD'\n",
     "name = 'cced_2024b_release'\n",
     "\n",
     "# unzip(s3, folder = folder, file = f\"{name}.shp.zip\")\n",
     "# cols = process_vector(s3, folder = folder, file = f\"{name}.shp\", crs=\"EPSG:3310\")\n",
     "# convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
     "cols = process_vector(s3, folder = folder, file = f\"{name}.shp\", crs=\"EPSG:4326\")\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols= cols)\n",
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols= cols, zoom = 8)\n",
     "\n",
     "name = 'cpad_2024b_release'\n",
     "# unzip(s3, folder = folder, file = f\"{name}.shp.zip\")\n",
     "# cols = process_vector(s3, folder = folder, file = f\"{name}.shp\", crs=\"EPSG:3310\")\n",
     "# convert_pmtiles(con, s3, folder = folder, file = f\"{name}.parquet\")\n",
     "cols = process_vector(s3, folder = folder, file = f\"{name}.shp\", crs=\"EPSG:4326\")\n",
-    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols= cols)"
+    "convert_h3(con, s3, folder = folder, file = f\"{name}.parquet\", cols= cols, zoom = 8)"
    ]
   }
  ],

preprocess/h3_utils.py

@@ -1,14 +1,63 @@
 from utils import *
+import re
 
-default_zoom = "8"
-max_h3_n = 1_000_000   # if est total H3 cells > this -> process in chunks
-chunk_n = 10_000     # chunk size (# of geoms)
-big_n = 10_000        # if geoms > big_n -> they are "big" and processed individually 
-batch_n = 5_000      # big geoms processed in batches of this size
+def convert_h3(con, s3, folder, file, cols, zoom):
+    """
+    Driver function to convert geometries to h3.
+    If no zoom levels exist -> compute from geometry at target zoom.
+    If lower zoom exists -> compute children from max available until target zoom.
+    """
+    cols = ", ".join(cols) if isinstance(cols, list) else cols
+    bucket, path = info(folder, file)
+    path, file = os.path.split(path)
+    name, ext = os.path.splitext(file)
+    name = name.replace('-', '')
+    print(f"Processing: {name}")
 
-def compute_h3(con, name, cols, zoom):
+    hex_paths = s3.list_objects(bucket, prefix=f"{path}/hex/", recursive=True)
+    zooms = []
+    # check what zooms exist 
+    for obj in hex_paths:
+        match = re.search(r"/zoom(\d{1,2})/", obj.object_name)
+        if match:
+            zooms.append(int(match.group(1)))
+            
+    if not zooms: # if no h3 files exist
+        print(f'No h3 files exists, computing {zoom} from geometry.')
+        con.read_parquet(f"s3://{bucket}/{path}/{file}", table_name=name)
+        h3_from_geom(con, name, cols, zoom)
+        con.sql(f'''
+            SELECT {cols}, UNNEST(h{zoom}) AS h{zoom}
+            FROM t2
+        ''').to_parquet(f"s3://{bucket}/{path}/hex/zoom{zoom}/{name}.parquet")
+
+    else: 
+        current_zoom = max(zooms)            
+        
+        if zoom in zooms:
+            print(f'Zoom {zoom} already exists!')
+            return 
+            
+        elif current_zoom < zoom: #compute child of most refined zoom level
+            print(f'Reading zoom {current_zoom}')
+            con.read_parquet(
+                f"s3://{bucket}/{path}/hex/zoom{current_zoom}/{name}.parquet",
+                table_name=f"h3_h{current_zoom}"
+            )
+            print(f'Computing {zoom} from {current_zoom}')
+            
+            for z in range(current_zoom + 1, zoom + 1):
+                print(f'Current zoom {z}')
+                h3_from_parent(con, z)
+                con.sql(f'''
+                    SELECT *, UNNEST(h3_cell_to_children(h{z-1}, {z})) AS h{z}
+                    FROM h3_h{z-1}
+                ''').to_parquet(f"s3://{bucket}/{path}/hex/zoom{z}/{name}.parquet")
+  
+    
+def h3_from_geom(con, name, cols, zoom):
     """
-    Computes hexes    
+    Computes hexes directly from geometry.
     """
     con.raw_sql(f'''
         CREATE OR REPLACE TEMP TABLE t2 AS
@@ -17,161 +66,14 @@ def compute_h3(con, name, cols, zoom):
             FROM {name}
         )
         SELECT {cols},
-               h3_polygon_wkt_to_cells_string(UNNEST(geom).geom, {zoom}) AS h{zoom}
+               h3_polygon_wkt_to_cells_string(ST_Force2D(UNNEST(geom).geom), {zoom}) AS h{zoom}
         FROM t1
     ''')
-    
-def check_size(con, name, zoom, sample_size):
-    """
-    Estimating size of geoms to decide if we need to process in chunks 
-    """
-    query = f"""
-        SELECT 
-            avg(len(h3_polygon_wkt_to_cells_string(ST_AsText(geom), {zoom}))::DOUBLE) AS avg_h3_len,
-            max(len(h3_polygon_wkt_to_cells_string(ST_AsText(geom), {zoom}))) AS max_h3_len,
-            count(*) AS total_rows
-        FROM {name}
-        USING SAMPLE {sample_size}
-    """
-    stats = con.sql(query).execute()
-    avg_len = stats.iloc[0]['avg_h3_len']
-    max_len = stats.iloc[0]['max_h3_len']
-    total_rows = con.table(name).count().execute()
-
-    est_total_h3 = avg_len * total_rows
-
-    print(f"Estimated total H3 cells: {est_total_h3:,.0f}")
-    print(f"Max H3 cells in one geometry: {max_len:,}")
-
-    return est_total_h3, max_len
-
-def chunk_large_geom(con, s3, bucket, path, name, zoom, big_n, batch_limit):
-    """
-    Individually processing large geoms (different from processing "chunks")
-    """
-    offset = 0
-    i = 0
-    while True:
-        relative_key = f"{path}/hex/zoom{zoom}/{name}_large_{i:03d}.parquet"
-        print(f"🟠 Checking large geometry batch {i} → {relative_key}")
-
-        if exists_on_s3(s3, folder="", file=relative_key):  # we pass relative_key as `file`
-            print(f"⏩ Skipping existing large batch: {relative_key}")
-            offset += batch_limit
-            i += 1
-            continue
 
-        print(f"📝 Writing large geometry batch {i} → {relative_key}")
-        q = con.sql(f'''
-            SELECT *, UNNEST(h{zoom}) AS h{zoom}
-            FROM t2
-            WHERE len(h{zoom}) > {big_n}
-            LIMIT {batch_limit} OFFSET {offset}
-        ''')
-
-        q.to_parquet(f"s3://{bucket}/{relative_key}")
-
-        if q.count().execute() == 0:
-            break
-
-        offset += batch_limit
-        i += 1
-
-    return i
-
-def join_large_geoms(con, s3, bucket, path, name, zoom):
-    """
-    If we had to process large geoms individually, join those datasets after conversion.
-    """
-    # check if any large files exist before trying to join
-    test_key = f"{path}/hex/zoom{zoom}/{name}_large_000.parquet"
 
-    if not exists_on_s3(s3, folder="", file=test_key):
-        print("✅ No large geometry chunks to join.")
-        return
-
-    # join if it exists 
+def h3_from_parent(con, zoom):
     con.raw_sql(f'''
-        COPY (
-            SELECT * FROM read_parquet('s3://{bucket}/{path}/hex/zoom{zoom}/{name}_large_*.parquet')
-        )
-        TO 's3://{bucket}/{path}/hex/zoom{zoom}/{name}_large.parquet'
-        (FORMAT PARQUET)
+        CREATE OR REPLACE TEMP TABLE h3_h{zoom} AS
+        SELECT *, UNNEST(h3_cell_to_children(h{zoom-1}, {zoom})) AS h{zoom}
+        FROM h3_h{zoom-1}
     ''')
-    
-
-def chunk_geom(con, s3, bucket, path, name, zoom, limit, batch_limit, big_n):
-    """
-    Processing files in chunks. 
-    """
-    offset = 0
-    i = 0
-    
-    while True:
-        chunk_path = f"{path}/hex/zoom{zoom}/{name}_chunk{i:03d}.parquet"
-        
-        if exists_on_s3(s3, folder="", file=chunk_path):  # relative path passed as file
-            print(f"⏩ Skipping existing chunk: {chunk_path}")
-            offset += limit
-            i += 1
-            continue
-
-        print(f"📝 Writing chunk {i} → {chunk_path}")
-        q = con.sql(f'''
-            SELECT *, UNNEST(h{zoom}) AS h{zoom}
-            FROM t2
-            WHERE len(h{zoom}) <= {big_n}
-            LIMIT {limit} OFFSET {offset}
-        ''')
-        q.to_parquet(f"s3://{bucket}/{chunk_path}")
-        if q.count().execute() == 0:
-            break
-        offset += limit
-        i += 1
-
-    # process large geometries using same threshold and limit
-    chunk_large_geom(con, s3, bucket, path, name, zoom, big_n, batch_limit)
-    join_large_geoms(con, s3, bucket, path, name, zoom)
-    return i
-
-
-
-def join_chunked(con, bucket, path, name, zoom):
-    """
-    If we had to chunk the data, join those datasets after conversion.
-    """
-    con.raw_sql(f'''
-        COPY (
-        SELECT * FROM read_parquet('s3://{bucket}/{path}/hex/zoom{zoom}/{name}_chunk*.parquet')
-        )
-        TO 's3://{bucket}/{path}/hex/zoom{zoom}/{name}.parquet'
-        (FORMAT PARQUET)
-        ''')
-
-def convert_h3(con, s3, folder, file, cols, zoom=default_zoom, limit=chunk_n, batch_limit = batch_n, big_n=big_n, max_h3_n = max_h3_n):
-    """
-    Driver function to convert geometries to h3
-    """
-    cols = ", ".join(cols) if isinstance(cols, list) else cols
-    bucket, path = info(folder, file)
-    path, file = os.path.split(path)
-    name, ext = os.path.splitext(file)
-    name = name.replace('-', '')
-
-    print(f"Processing: {name}")
-    con.read_parquet(f"s3://{bucket}/{path}/{file}", table_name=name)
-
-    # Decide to chunk or not
-    est_total, max_per_geom = check_size(con, name, zoom, sample_size=100)
-    if est_total > max_h3_n or max_per_geom > big_n:
-        print("Chunking due to estimated size")
-        compute_h3(con, name, cols, zoom)
-        chunk_geom(con, s3, bucket, path, name, zoom, limit, batch_limit, big_n)
-        join_chunked(con, bucket, path, name, zoom)
-    else:
-        print("Writing single output")
-        compute_h3(con, name, cols, zoom)
-        con.sql(f'''
-            SELECT *, UNNEST(h{zoom}) AS h{zoom}
-            FROM t2
-        ''').to_parquet(f"s3://{bucket}/{path}/hex/zoom{zoom}/{name}.parquet")

preprocess/preprocess.ipynb

@@ -10,7 +10,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "f7e6298c-d886-432a-a1b7-c3fee914c24f",
    "metadata": {
     "editable": true,
@@ -48,12 +48,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "63dd33b8-6d3c-4852-9899-6ed5775d19c0",
    "metadata": {},
    "outputs": [],
    "source": [
-    "def get_url(folder, file, base_folder = 'CBN-data'):\n",
+    "def get_url(folder, file, base_folder = 'CBN'):\n",
     "    minio = 'https://minio.carlboettiger.info/'\n",
     "    bucket = 'public-ca30x30'\n",
     "    if base_folder is None:\n",
@@ -80,7 +80,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "13214bbe-3a74-4247-981f-5a6eb6c486f5",
    "metadata": {},
    "outputs": [],
@@ -90,7 +90,7 @@
     "# ca_raw_parquet = 'ca_areas.parquet'\n",
     "\n",
     "# Boundary of CA, used to computed 'non-conserved' areas\n",
-    "ca_boundary_parquet = get_url('Preprocessing','ca_boundary.parquet',base_folder = None)\n",
+    "ca_boundary_parquet = get_url('CA_Nature/2024/Preprocessing','ca_boundary.parquet',base_folder = None)\n",
     "\n",
     "# newly protected areas \n",
     "newly_protected = get_url('Progress_data_new_protection/Newly_counted_lands','newly_counted_lands_2024.parquet')\n",
@@ -167,43 +167,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "0f9666d1-7c2b-45af-9399-e4189bba34f5",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "52ef18913f17417299860d91e36e9dbd",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "CPU times: user 4min 28s, sys: 6.1 s, total: 4min 34s\n",
-      "Wall time: 2min 18s\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "<minio.helpers.ObjectWriteResult at 0x7ff0943c7710>"
-      ]
-     },
-     "execution_count": 7,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "%%time \n",
     "# match CA Nature schema \n",
@@ -241,7 +208,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "a3d4f189-1563-4868-9f1f-64d67569df27",
    "metadata": {},
    "outputs": [],
@@ -298,7 +265,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "a59c976b-3c36-40f9-a15b-cefcd155c647",
    "metadata": {},
    "outputs": [],
@@ -344,58 +311,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "4d6177e2-8ece-4eb9-acc2-5fb5c5beb8bb",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "09f24f1359a84ae2a4b69360cc8e852b",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c10ce980d24e45b6bad9b8a70c176f2c",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/opt/conda/lib/python3.12/site-packages/ibis/common/deferred.py:408: FutureWarning: `Value.case` is deprecated as of v10.0.0; use value.cases() or ibis.cases()\n",
-      "  return func(*args, **kwargs)\n"
-     ]
-    },
-    {
-     "ename": "NameError",
-     "evalue": "name 'non_conserved' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
-      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
-      "\u001b[36mFile \u001b[39m\u001b[32m<timed exec>:50\u001b[39m\n",
-      "\u001b[31mNameError\u001b[39m: name 'non_conserved' is not defined"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "%%time \n",
     "counties = con.read_parquet('../CA_counties.parquet')\n",
@@ -454,7 +373,7 @@
     "gdf = all_data.execute()\n",
     "\n",
     "gdf.set_crs(\"epsg:3310\").to_parquet(ca_base_parquet)\n",
-    "s3.fput_object(\"public-ca30x30\", 'Preprocessing/'+ca_base_parquet, ca_base_parquet) "
+    "s3.fput_object(\"public-ca30x30\", 'CA_Nature/2024/Preprocessing/'+ca_base_parquet, ca_base_parquet) "
    ]
   },
   {
@@ -485,7 +404,7 @@
     "\n",
     "def get_habitat_type(fieldname):\n",
     "    aux_xml_path = 'fveg22_1_processed.tif.aux.xml'\n",
-    "    s3.fget_object('public-ca30x30','CBN-data/Habitat/'+aux_xml_path, aux_xml_path)\n",
+    "    s3.fget_object('public-ca30x30','CBN/Habitat/'+aux_xml_path, aux_xml_path)\n",
     "    tree = ET.parse(aux_xml_path)\n",
     "    root = tree.find(\".//GDALRasterAttributeTable\")\n",
     "    field_names = [f.find(\"Name\").text for f in root.findall(\"FieldDefn\")]\n",

preprocess/utils.py

@@ -15,7 +15,7 @@ from shapely.geometry import shape
 import numpy as np
 
 
-def info(folder, file, bucket = "public-ca30x30", base_folder = 'CBN-data/'):
+def info(folder, file, bucket = "public-ca30x30", base_folder = 'CBN/'):
     """
     Extract minio path to upload/download data 
     """
@@ -77,22 +77,10 @@ def process_raster(s3, folder, file, file_name = None):
     """
     if file_name:
         file = file_name
-    # output_file = reproject_raster(file)
-    # upload(s3, folder, output_file)
-    # output_cog_file = make_cog(output_file)
-    # upload(s3, folder, output_cog_file)
-    # output_vector, cols  = make_vector(output_file)
-    # upload(s3, folder, output_vector)
-
     name, ext = os.path.splitext(file)
     output_file = f"{name}_processed{ext}"
-
     output_cog_file = f"{name}_processed_COG{ext}"
-
     output_vector_file = f"{name}_processed.parquet"
-    print(output_file)
-    print(output_cog_file)
-    print(output_vector_file)
     # Reproject raster
     if not exists_on_s3(s3, folder, output_file):
         output_file = reproject_raster(file)
@@ -183,7 +171,6 @@ def make_vector(input_file, crs="EPSG:4326"):
         gdf.to_crs(crs, inplace=True)
 
     gdf.to_parquet(output_file)
-    print(gdf)
     return output_file, gdf.drop('geom',axis = 1).columns.to_list()
 
 def filter_raster(s3, folder, file, percentile):
@@ -226,9 +213,6 @@ def exists_on_s3(s3, folder, file):
     Check if a file exists on S3
     """
     bucket, path = info(folder, file)
-    print(bucket)
-    print(path)
-    
     try:
         s3.stat_object(bucket, path)
         return True