 geo_point field does not support GeoJson which can be a PITA if your data is in such a format. Indexing it as a geo_shape is an option but very wasteful.
server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
final GeoPoint to = singleValues.geoPointValue();
                    return distance.calculate(from.lat(), from.lon(), to.lat(), to.lon(), unit);
                }
            });
        } else {
            return new SortingNumericDoubleValues() {
                @Override
                public boolean advanceExact(int target) throws IOException {
                    if (geoPointValues.advanceExact(target)) {
                        resize(geoPointValues.docValueCount() * fromPoints.length);
                        int v = 0;
                        for (int i = 0; i < geoPointValues.docValueCount(); ++i) {
                            final GeoPoint point = geoPointValues.nextValue();
                            for (GeoPoint from : fromPoints) {
                                values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit);
                                v++;
                            }
                        }
                        sort();
                        return true;
                    } else {
                        return false;
                    }
                }
            };
        }
    }
    private GeoUtils() {}
}