### Apache Airflow version

2.7.0

### What happened

We recently updated our Airflow server to 2.7.0 (from 2.6.0) and moved from a local PostGres instance to one located in AWS RDS, as well as switched from X86_64 arch to ARM64 (Amazon Graviton2 processor). We had some DAGs that passed pendulum.DateTimes as XCOMs that used to work on the old server that now fail with the following error: 

```
[2023-09-19, 15:28:23 UTC] {abstractoperator.py:696} ERROR - Exception rendering Jinja template for task 'apply_bonuses_to_new_shifts', field 'op_args'. Template: (XComArg(<Task(_PythonDecoratedOperator): update_early_bird_eligible_shifts>),)
Traceback (most recent call last):
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/zoneinfo/reader.py", line 50, in read_for
    file_path = pytzdata.tz_path(timezone)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pytzdata/__init__.py", line 74, in tz_path
    raise TimezoneNotFound('Timezone {} not found at {}'.format(name, filepath))
pytzdata.exceptions.TimezoneNotFound: Timezone EDT not found at /home/airflow/dagger/venv/lib64/python3.9/site-packages/pytzdata/zoneinfo/EDT
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/abstractoperator.py", line 688, in _do_render_template_fields
    rendered_content = self.render_template(
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/template/templater.py", line 162, in render_template
    return tuple(self.render_template(element, context, jinja_env, oids) for element in value)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/template/templater.py", line 162, in <genexpr>
    return tuple(self.render_template(element, context, jinja_env, oids) for element in value)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/template/templater.py", line 158, in render_template
    return value.resolve(context)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/utils/session.py", line 77, in wrapper
    return func(*args, session=session, **kwargs)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/xcom_arg.py", line 413, in resolve
    result = ti.xcom_pull(
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/utils/session.py", line 74, in wrapper
    return func(*args, **kwargs)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/taskinstance.py", line 2562, in xcom_pull
    return XCom.deserialize_value(first)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/xcom.py", line 693, in deserialize_value
    return BaseXCom._deserialize_value(result, False)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/xcom.py", line 686, in _deserialize_value
    return json.loads(result.value.decode("UTF-8"), cls=XComDecoder, object_hook=object_hook)
  File "/usr/lib64/python3.9/json/__init__.py", line 359, in loads
    return cls(**kw).decode(s)
  File "/usr/lib64/python3.9/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/lib64/python3.9/json/decoder.py", line 353, in raw_decode
    obj, end = self.scan_once(s, idx)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/utils/json.py", line 117, in object_hook
    return deserialize(dct)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/serialization/serde.py", line 253, in deserialize
    return _deserializers[classname].deserialize(classname, version, deserialize(value))
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/serialization/serializers/datetime.py", line 70, in deserialize
    return DateTime.fromtimestamp(float(data[TIMESTAMP]), tz=timezone(data[TIMEZONE]))
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/__init__.py", line 37, in timezone
    tz = _Timezone(name, extended=extended)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/timezone.py", line 40, in __init__
    tz = read(name, extend=extended)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/zoneinfo/__init__.py", line 9, in read
    return Reader(extend=extend).read_for(name)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/zoneinfo/reader.py", line 52, in read_for
    raise InvalidTimezone(timezone)
pendulum.tz.zoneinfo.exceptions.InvalidTimezone: Invalid timezone "EDT"
[2023-09-19, 15:28:23 UTC] {taskinstance.py:1943} ERROR - Task failed with exception
Traceback (most recent call last):
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/zoneinfo/reader.py", line 50, in read_for
    file_path = pytzdata.tz_path(timezone)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pytzdata/__init__.py", line 74, in tz_path
    raise TimezoneNotFound('Timezone {} not found at {}'.format(name, filepath))
pytzdata.exceptions.TimezoneNotFound: Timezone EDT not found at /home/airflow/dagger/venv/lib64/python3.9/site-packages/pytzdata/zoneinfo/EDT
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/taskinstance.py", line 1518, in _run_raw_task
    self._execute_task_with_callbacks(context, test_mode, session=session)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/taskinstance.py", line 1646, in _execute_task_with_callbacks
    task_orig = self.render_templates(context=context)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/taskinstance.py", line 2291, in render_templates
    original_task.render_template_fields(context)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/baseoperator.py", line 1244, in render_template_fields
    self._do_render_template_fields(self, self.template_fields, context, jinja_env, set())
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/utils/session.py", line 77, in wrapper
    return func(*args, session=session, **kwargs)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/abstractoperator.py", line 688, in _do_render_template_fields
    rendered_content = self.render_template(
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/template/templater.py", line 162, in render_template
    return tuple(self.render_template(element, context, jinja_env, oids) for element in value)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/template/templater.py", line 162, in <genexpr>
    return tuple(self.render_template(element, context, jinja_env, oids) for element in value)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/template/templater.py", line 158, in render_template
    return value.resolve(context)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/utils/session.py", line 77, in wrapper
    return func(*args, session=session, **kwargs)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/xcom_arg.py", line 413, in resolve
    result = ti.xcom_pull(
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/utils/session.py", line 74, in wrapper
    return func(*args, **kwargs)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/taskinstance.py", line 2562, in xcom_pull
    return XCom.deserialize_value(first)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/xcom.py", line 693, in deserialize_value
    return BaseXCom._deserialize_value(result, False)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/models/xcom.py", line 686, in _deserialize_value
    return json.loads(result.value.decode("UTF-8"), cls=XComDecoder, object_hook=object_hook)
  File "/usr/lib64/python3.9/json/__init__.py", line 359, in loads
    return cls(**kw).decode(s)
  File "/usr/lib64/python3.9/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/lib64/python3.9/json/decoder.py", line 353, in raw_decode
    obj, end = self.scan_once(s, idx)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/utils/json.py", line 117, in object_hook
    return deserialize(dct)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/serialization/serde.py", line 253, in deserialize
    return _deserializers[classname].deserialize(classname, version, deserialize(value))
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/airflow/serialization/serializers/datetime.py", line 70, in deserialize
    return DateTime.fromtimestamp(float(data[TIMESTAMP]), tz=timezone(data[TIMEZONE]))
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/__init__.py", line 37, in timezone
    tz = _Timezone(name, extended=extended)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/timezone.py", line 40, in __init__
    tz = read(name, extend=extended)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/zoneinfo/__init__.py", line 9, in read
    return Reader(extend=extend).read_for(name)
  File "/home/airflow/dagger/venv/lib64/python3.9/site-packages/pendulum/tz/zoneinfo/reader.py", line 52, in read_for
    raise InvalidTimezone(timezone)
pendulum.tz.zoneinfo.exceptions.InvalidTimezone: Invalid timezone "EDT"
```

However, the DAG itself makes no mention of EDT - it does the following:

```
last_datetime.in_tz("America/New_York")
```

I've identified the issue as how these XCOMs are being serialized - one our old server, they were being serialized as ISO timestamps:
![image](https://github.com/apache/airflow/assets/29555644/f0027b3a-2245-4529-aa52-7e9cff8cd009)

Now, however, they're being serialized like this:

![image](https://github.com/apache/airflow/assets/29555644/485ac5a8-eb29-4304-96f3-f6612b36d60c)

This is unexpected, and also causes problems because `EDT` is not an IANA timezone, which prevents `pendulum` from deserializing it in the task that accepts this XCOM. 


### What you think should happen instead

_No response_

### How to reproduce

I think this is reproducible by creating a DAG that has a task that converts a `pendulum.DateTime` to `America/New_York` and passes it as an XCOM to another task. 

```python
from datetime import datetime, timedelta
from typing import Dict, List, Optional

from airflow.decorators import dag, task
from airflow.utils.trigger_rule import TriggerRule
from pendulum.datetime import DateTime

@task()
def task_one(
    data_interval_end: Optional[DateTime] = None,
) -> DateTime:
    return data_interval_end.in_tz("America/New_York")

# this task will error out
@task()
def task_two(last_added_date: DateTime) -> None:
    pass

@dag(
    default_args=default_args,
    schedule="*/5 * * * *",
    start_date=datetime(2023, 7, 25, 18, 0, 0),
)
def dag() -> None:
    last_added_datetime = task_one()
    task_two(last_added_datetime)

dag()
```

### Operating System

Ubuntu

### Versions of Apache Airflow Providers

_No response_

### Deployment

Virtualenv installation

### Deployment details

_No response_

### Anything else

_No response_

### Are you willing to submit PR?

- [ ] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)

from datetime import date, datetime, timedelta
    if isinstance(o, datetime):
        qn = qualname(o)
        if is_naive(o):
            o = convert_to_utc(o)
        tz = o.tzname()
        return {TIMESTAMP: o.timestamp(), TIMEZONE: tz}, qn, __version__, True
    if isinstance(o, date):
        return o.isoformat(), qualname(o), __version__, True
    if isinstance(o, timedelta):
        return o.total_seconds(), qualname(o), __version__, True
    return "", "", 0, False
def deserialize(classname: str, version: int, data: dict | str) -> datetime.date | datetime.timedelta:
    import datetime
    from pendulum import DateTime
    from pendulum.tz import timezone
    if classname == qualname(datetime.datetime) and isinstance(data, dict):
        return datetime.datetime.fromtimestamp(float(data[TIMESTAMP]), tz=timezone(data[TIMEZONE]))
    if classname == qualname(DateTime) and isinstance(data, dict):
        return DateTime.fromtimestamp(float(data[TIMESTAMP]), tz=timezone(data[TIMEZONE]))
    if classname == qualname(datetime.timedelta) and isinstance(data, (str, float)):
        return datetime.timedelta(seconds=float(data))
    if classname == qualname(datetime.date) and isinstance(data, str):
        return datetime.date.fromisoformat(data)
    raise TypeError(f"unknown date/time format {classname}")