wip; ecoregion + filter updates

separated gap 3 and 4, ecoregions for non-conserved land, synced 30x30 status/gap filters, and fixed pmtiles filters

app/app.py

@@ -25,7 +25,8 @@ current_tables = con.list_tables()
 if "mydata" not in set(current_tables):
     tbl = con.read_parquet(ca_parquet)
     con.create_table("mydata", tbl)
-    
+
+ 
 ca = con.table("mydata")
 
     
@@ -190,12 +191,22 @@ def run_sql(query,color_choice):
     elif ("id" and "geom" in result.columns): 
         style = get_pmtiles_style_llm(style_options[color_choice], result["id"].tolist())
         legend_d = {cat: color for cat, color in style_options[color_choice]['stops']}
-
+        position = 'bottom-left'
+        fontsize = 15
+        bg_color = 'white'
+        
         # shorten legend for ecoregions 
         if color_choice == "Ecoregion":
+            legend_d = {key.replace("Northern California", "NorCal"): value for key, value in legend_d.items()} 
+            legend_d = {key.replace("Southern California", "SoCal"): value for key, value in legend_d.items()} 
+            legend_d = {key.replace("Southeastern", "SE."): value for key, value in legend_d.items()} 
+            legend_d = {key.replace("and", "&"): value for key, value in legend_d.items()} 
             legend_d = {key.replace("California", "CA"): value for key, value in legend_d.items()} 
-            
-        m.add_legend(legend_dict=legend_d, position='bottom-left')
+            legend_d = {key.replace("Northwestern", "NW."): value for key, value in legend_d.items()} 
+            bg_color = 'rgba(255, 255, 255, 0.6)'
+            fontsize = 12
+        
+        m.add_legend(legend_dict = legend_d, position = position, bg_color = bg_color, fontsize = fontsize)
         m.add_pmtiles(ca_pmtiles, style=style, opacity=alpha, tooltip=True, fit_bounds=True)
         m.fit_bounds(result.total_bounds.tolist())    
         result = result.drop('geom',axis = 1) #printing to streamlit so I need to drop geom
@@ -350,9 +361,14 @@ with st.sidebar:
 
     st.divider()
     st.markdown('<p class = "medium-font-sidebar"> Filters:</p>', help = "Apply filters to adjust what data is shown on the map.", unsafe_allow_html= True)
+    for col,val in style_options.items():
+        for name in val['stops'][0]:
+            key = val['property']+str(name)
+            st.session_state[key] = default_gap.get(name, True)
+
     for label in style_options: # get selected filters (based on the buttons selected)
         with st.expander(label):  
-            if label == "GAP Code": # gap code 1 and 2 are on by default
+            if label in ["GAP Code","30x30 Status"]: # gap code 1 and 2 are on by default
                 opts = getButtons(style_options, label, default_gap)
             else: # other buttons are not on by default.
                 opts = getButtons(style_options, label) 
@@ -365,7 +381,7 @@ with st.sidebar:
         else: 
             filter_cols = []
             filter_vals = []
-            
+
     st.divider()
     st.markdown("""
     <p class="medium-font-sidebar">
@@ -376,12 +392,23 @@ with st.sidebar:
 if 'out' not in locals():
     style = get_pmtiles_style(style_options[color_choice], alpha, filter_cols, filter_vals)
     legend_d = {cat: color for cat, color in style_options[color_choice]['stops']}
-
+    position = 'bottom-left'
+    fontsize = 15
+    bg_color = 'white'
+    
     # shorten legend for ecoregions 
     if color_choice == "Ecoregion":
+        legend_d = {key.replace("Northern California", "NorCal"): value for key, value in legend_d.items()} 
+        legend_d = {key.replace("Southern California", "SoCal"): value for key, value in legend_d.items()} 
+        legend_d = {key.replace("Southeastern", "SE."): value for key, value in legend_d.items()} 
+        legend_d = {key.replace("and", "&"): value for key, value in legend_d.items()} 
         legend_d = {key.replace("California", "CA"): value for key, value in legend_d.items()} 
+        legend_d = {key.replace("Northwestern", "NW."): value for key, value in legend_d.items()} 
+        bg_color = 'rgba(255, 255, 255, 0.6)'
+        fontsize = 12
+            
         
-    m.add_legend(legend_dict = legend_d, position = 'bottom-left')
+    m.add_legend(legend_dict = legend_d, position = position, bg_color = bg_color, fontsize = fontsize)
     m.add_pmtiles(ca_pmtiles, style=style, name="CA", opacity=alpha, tooltip=True, fit_bounds=True)
     
 
@@ -408,11 +435,12 @@ colors = (
 # get summary tables used for charts + printed table 
 # df - charts; df_tab - printed table (omits colors) 
 if 'out' not in locals():
-    df,df_tab = summary_table(ca, column, colors, filter_cols, filter_vals, colorby_vals)
+    df,df_tab,df_percent = summary_table(ca, column, colors, filter_cols, filter_vals, colorby_vals)
+    total_percent = df_percent.percent_protected.sum().round(2)
+
 else:
     df = summary_table_sql(ca, column, colors, ids)
-
-total_percent = df.percent_protected.sum().round(2)
+    total_percent = df.percent_protected.sum().round(2)
 
 
 # charts displayed based on color_by variable

app/utils.py

@@ -14,6 +14,7 @@ import sqlalchemy
 import pathlib
 from typing import Optional
 from functools import reduce
+from itertools import chain
 
 from variables import *
 
@@ -52,13 +53,17 @@ def summary_table(ca, column, colors, filter_cols, filter_vals,colorby_vals): #
         filter_cols.append(column)
         filters.append(getattr(_, column).isin(colorby_vals[column])) 
     combined_filter = reduce(lambda x, y: x & y, filters) #combining all the filters into ibis filter expression 
-    
+
+    df_percent = get_summary(ca, combined_filter, [column], colors) # df used for charts 
+
     if column == "status": #need to include non-conserved in summary stats 
         combined_filter = (combined_filter) | (_.status.isin(['30x30-conserved','other-conserved','non-conserved']))
-        
+
     df = get_summary(ca, combined_filter, [column], colors) # df used for charts 
+
     df_tab = get_summary(ca, combined_filter, filter_cols, colors = None) #df used for printed table
-    return df, df_tab 
+
+    return df, df_tab, df_percent
 
 
 
@@ -120,19 +125,55 @@ def bar_chart(df, x, y, title): #display summary stats for color_by column
         ).properties(width="container", height=height, title = title)
     return chart
 
-
-def getButtons(style_options, style_choice, default_gap=None): #finding the buttons selected to use as filters 
+def sync_checkboxes(source):
+    # gap 1 and gap 2 on -> 30x30-conserved on
+    if source in ["gap_code1", "gap_code2"]:
+        st.session_state['status30x30-conserved'] = st.session_state.gap_code1 and st.session_state.gap_code2
+
+    # 30x30-conserved on -> gap 1 and gap 2 on
+    elif source == "status30x30-conserved":
+        st.session_state.gap_code1 = st.session_state['status30x30-conserved']
+        st.session_state.gap_code2 = st.session_state['status30x30-conserved']
+
+    # other-conserved on <-> gap 3 on
+    elif source == "gap_code3":
+        st.session_state["statusother-conserved"] = st.session_state.gap_code3
+        rerun_needed = True
+    elif source == "statusother-conserved":
+        if "gap_code3" in st.session_state and st.session_state["statusother-conserved"] != st.session_state.gap_code3:
+            st.session_state.gap_code3 = st.session_state["statusother-conserved"]
+            rerun_needed = True  # Ensure UI updates
+
+    # unknown on <-> gap 4 on
+    elif source == "gap_code4":
+        st.session_state.statusunknown = st.session_state.gap_code4
+        rerun_needed = True
+    elif source == "statusunknown":
+        if "gap_code4" in st.session_state and st.session_state.statusunknown != st.session_state.gap_code4:
+            st.session_state.gap_code4 = st.session_state.statusunknown
+            rerun_needed = True
+
+    # non-conserved on <-> gap 0 
+    elif source == "gap_code0":
+        st.session_state['statusnon-conserved'] = st.session_state.gap_code0
+        rerun_needed = True
+    elif source == "statusnon-conserved":
+        if "gap_code0" in st.session_state and st.session_state['statusnon-conserved'] != st.session_state.gap_code0:
+            st.session_state.gap_code0 = st.session_state['statusnon-conserved']
+            rerun_needed = True
+
+
+def getButtons(style_options, style_choice, default_gap=None):
     column = style_options[style_choice]['property']
-    opts = [style[0] for style in style_options[style_choice]['stops']]   
-    default_gap = default_gap or {}  
-    buttons = {
-        name: st.checkbox(f"{name}", value=default_gap.get(name, True), key=column + str(name))
-        for name in opts
-    }
-    filter_choice = [key for key, value in buttons.items() if value]  # return only selected
-    d = {}
-    d[column] = filter_choice
-    return d
+    opts = [style[0] for style in style_options[style_choice]['stops']]
+    default_gap = default_gap or {}
+    buttons = {}
+    for name in opts:
+        key = column + str(name)
+        buttons[name] = st.checkbox(f"{name}", value=st.session_state[key], key=key, on_change = sync_checkboxes, args = (key,))
+    filter_choice = [key for key, value in buttons.items() if value]
+    return {column: filter_choice}
+    
 
 
 def getColorVals(style_options, style_choice): 
@@ -149,6 +190,10 @@ def get_pmtiles_style(paint, alpha, filter_cols, filter_vals):
     for col, val in zip(filter_cols, filter_vals):
         filters.append(["match", ["get", col], val, True, False])
     combined_filters = ["all"] + filters
+    
+    if "non-conserved" in list(chain.from_iterable(filter_vals)):
+       combined_filters = ["any", combined_filters, ["match", ["get", "status"], ["non-conserved"],True, False]]
+        
     style = {
         "version": 8,
         "sources": {

app/variables.py

@@ -1,6 +1,7 @@
 # urls for main layer 
-ca_parquet = "https://huggingface.co/datasets/boettiger-lab/ca-30x30/resolve/8d5d938c04d3206e6bfb04055b5e779c4c28222f/ca-30x30.parquet"
-ca_pmtiles = "https://huggingface.co/datasets/boettiger-lab/ca-30x30/resolve/c58913a279d13c414722c4299b0e0867e923946a/ca-30x30.pmtiles"
+ca_parquet = "https://huggingface.co/datasets/boettiger-lab/ca-30x30/resolve/38af68979644f52ac928c5e41c81ec4d93468eef/ca-30x30.parquet"
+ca_pmtiles = "https://huggingface.co/datasets/boettiger-lab/ca-30x30/resolve/e283bb63ee76dd5acd2d187029a80ab6a011886b/ca-30x30.pmtiles"
+
 
 ca_area_acres = 1.014e8 #acres 
 style_choice = "GAP Status Code"
@@ -41,6 +42,9 @@ default_gap = {
     0: False,
     3: False,
     4: False,
+    "other-conserved":False,
+    "unknown":False,
+    "non-conserved":False
 }
 
 # Maplibre styles. (should these be functions?)
@@ -59,7 +63,8 @@ manager = {
         ['Tribal', tribal_color],
         ['Private', private_color],
         ['HOA', hoa_color],
-    ]
+    ],
+    'default': white
 }
 
 easement = {
@@ -68,7 +73,8 @@ easement = {
     'stops': [
         ['True', private_access_color],
         ['False', public_access_color],
-    ]
+    ],
+    'default': white
 }
 
 year = {
@@ -77,7 +83,8 @@ year = {
     'stops': [
         ['pre-2024', year2023_color],
         ['2024', year2024_color],
-    ]
+    ],
+    'default': white
 }
 
 access = {
@@ -88,7 +95,8 @@ access = {
         ['No Public Access', private_access_color],
         ['Unknown Access', "#bbbbbb"],
         ['Restricted Access', tribal_color],
-    ]
+    ],
+    'default': white
 }
 
 gap = {
@@ -97,22 +105,25 @@ gap = {
     'stops': [
         [1, "#26633d"],
         [2, "#879647"],
-        [3, "#EE4B2B"],
-        [4, "#BF40BF"]
-    ]
+        [3, "#bdcf72"],
+        [4, "#6d6e6d"]
+    ],
+    'default': white
 }
 
 status = {
     'property': 'status',
     'type': 'categorical',
     'stops': [
-        ['30x30-conserved', "#26633d"],
-        ['other-conserved', "#879647"],
-        ['non-conserved', white]
-    ]
+        ['30x30-conserved', "#56711f"],
+        ['other-conserved', "#b6ce7a"],
+        ['unknown', "#e5efdb"],
+        ['non-conserved', "#e1e1e1"]
+    ],
 }
 
 
+
 ecoregion = {
     'property': 'ecoregion',
     'type': 'categorical',
@@ -137,14 +148,14 @@ ecoregion = {
         ['Northern California Coast', "#c7c7c7"],
         ['Great Valley (North)', "#dbdb8d"],
         ['Central California Coast', "#9edae5"],
-    ]
+    ],
+    'default': white
 }
 
-
 style_options = {
     "Year": year,
-    "GAP Code": gap,
     "30x30 Status": status,
+    "GAP Code": gap,
     "Ecoregion": ecoregion,
     "Manager Type": manager,
     "Easement": easement,
@@ -252,8 +263,8 @@ svi_style = {
 
 select_column = {
     "Year": "established",
-    "GAP Code": "gap_code",
     "30x30 Status":  "status",
+    "GAP Code": "gap_code",
     "Ecoregion":  "ecoregion",
     "Manager Type": "manager_type",
     "Easement": "easement",

preprocess/preprocess.ipynb

@@ -37,8 +37,13 @@
     "ca_boundary_shape = \"../data/ca_shape\"\n",
     "ca_boundary_parquet = path + \"ca_boundary.parquet\"\n",
     "\n",
+    "# Ecoregions\n",
+    "ca_ecoregions_shape = \"../data/ecoregions/ACE_Ecoregions_BaileyDerived_2022.shp\"\n",
+    "ca_ecoregions_parquet = path + \"ace_ecoregions.parquet\"\n",
+    "\n",
     "# file to save non-conserved areas; costly operation so we save results \n",
     "ca_nonconserved_parquet = path + \"ca-30x30-nonconserved-500m-simplified.parquet\" \n",
+    "ca_nonconserved_eco_parquet = path + \"ca-30x30-nonconserved-500m-simplified-eco.parquet\" \n",
     "\n",
     "# temp file used to compute zonal stats: has conserved + non-conserved areas \n",
     "ca_temp_parquet = path + \"ca-30x30-temp.parquet\"  \n",
@@ -148,6 +153,88 @@
     "nonconserved.execute().to_parquet(ca_nonconserved_parquet)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "845eb0ed-3392-4346-9b7e-959cd97a274f",
+   "metadata": {},
+   "source": [
+    "#### Get ecoregions - convert them to parquet"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b43fee2c-b8c4-4076-b87b-089676031165",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eco = gpd.read_file(ca_ecoregions_shape)\n",
+    "eco.to_parquet(ca_ecoregions_parquet)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b5689850-80fa-4cc9-87e9-46074b8d9107",
+   "metadata": {},
+   "source": [
+    "#### Compute ecoregion for non-conserved areas"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "070bbdde-b141-4a63-8f8a-984dd01fd51a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3c217929b7744164a99f6e2314366359",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "con = ibis.duckdb.connect(extensions=[\"spatial\"])\n",
+    "\n",
+    "eco = con.read_parquet(ca_ecoregions_parquet)\n",
+    "non = con.read_parquet(ca_nonconserved_parquet)\n",
+    "\n",
+    "con.create_table(\"eco\", eco.select(\"ECOREGION_\",\"geometry\"), overwrite = True)\n",
+    "con.create_table(\"non\", non, overwrite = True)\n",
+    "\n",
+    "# split up the non-conserved areas by ecoregions\n",
+    "con.con.execute('''\n",
+    "CREATE TABLE non_conserved_eco AS\n",
+    "SELECT \n",
+    "    non.*, \n",
+    "    eco.ECOREGION_ AS ecoregion,\n",
+    "    ST_Intersection(non.geom, eco.geometry) AS geom  -- Split non into ecoregions\n",
+    "FROM non\n",
+    "JOIN eco \n",
+    "ON ST_Intersects(non.geom, eco.geometry)\n",
+    "WHERE ST_GeometryType(ST_Intersection(non.geom, eco.geometry)) IN ('POLYGON', 'MULTIPOLYGON');\n",
+    "''')\n",
+    "\n",
+    "\n",
+    "# save to parquet file so we don't have to run this again\n",
+    "non_eco = (con.table(\"non_conserved_eco\")\n",
+    "           .drop('geom')\n",
+    "           .rename(geom = \"geom_1\")\n",
+    "           .mutate(geom = ST_MakeValid(_.geom))          \n",
+    "          )\n",
+    "\n",
+    "non_conserved_eco = non_eco.execute()\n",
+    "non_conserved_eco.to_parquet(ca_nonconserved_eco_parquet)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "ce52b1e0-027e-4915-9e7b-e51e946560ed",
@@ -165,7 +252,7 @@
    "source": [
     "# match CA Nature schema \n",
     "nonconserved_clean = (\n",
-    "    con.read_parquet(ca_nonconserved_parquet)\n",
+    "    con.read_parquet(ca_nonconserved_eco_parquet)\n",
     "    .cast({\"geom\": \"geometry\"})\n",
     "    .mutate(established = ibis.null(), gap_code = 0, name = ibis.literal(\"Non-Conserved Areas\"),\n",
     "            access_type = ibis.null(), manager = ibis.null(), manager_type = ibis.null(),\n",
@@ -250,7 +337,7 @@
     "      .mutate(easement=_.Easement.cast(\"string\").substitute({\"0\": \"False\", \"1\": \"True\"}))\n",
     "      .mutate(status=_.gap_code.cast(\"string\")\n",
     "              .substitute({\"1\": \"30x30-conserved\", \"2\": \"30x30-conserved\", \"3\": \"other-conserved\", \n",
-    "                           \"4\": \"other-conserved\"}))\n",
+    "                           \"4\": \"unknown\"}))\n",
     "      .select(_.established, _.gap_code, _.status, _.name, _.access_type, _.manager, _.manager_type,\n",
     "              _.ecoregion, _.easement, _.acres, _.id, _.type, _.geom)\n",
     "      .union(nonconserved_clean)\n",
@@ -379,7 +466,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "aade11d9-87b9-403d-bad1-3069663807a9",
    "metadata": {},
    "outputs": [],
@@ -483,7 +570,7 @@
     "#to use PMTiles, need to convert to geojson\n",
     "ca_geojson = (con\n",
     "            .read_parquet(ca_parquet)\n",
-    "            .filter(_.status != 'non-conserved') #omitting the non-conserved to only for pmtiles  \n",
+    "            # .filter(_.status != 'non-conserved') #omitting the non-conserved to only for pmtiles  \n",
     "            )\n",
     "\n",
     "#can't go directly from parquet -> pmtiles, need to go parquet -> geojson -> pmtiles \n",