Upload AestheticsPredictorV1

config.json

@@ -1,10 +1,10 @@
 {
-  "_name_or_path": "openai/clip-vit-base-patch16",
   "architectures": [
     "AestheticsPredictorV1"
   ],
   "attention_dropout": 0.0,
   "auto_map": {
+    "AutoConfig": "configuration_predictor.AestheticsPredictorConfig",
     "AutoModel": "modeling_v1.AestheticsPredictorV1"
   },
   "dropout": 0.0,
@@ -15,7 +15,7 @@
   "initializer_range": 0.02,
   "intermediate_size": 3072,
   "layer_norm_eps": 1e-05,
-  "model_type": "clip_vision_model",
+  "model_type": "aesthetics_predictor",
   "num_attention_heads": 12,
   "num_channels": 3,
   "num_hidden_layers": 12,

modeling_v1.py

@@ -54,8 +54,15 @@ class AestheticsPredictorV1(CLIPVisionModelWithProjection):
         )
 
 
-def convert_from_openai_clip(openai_model_name: str) -> AestheticsPredictorV1:
-    model = AestheticsPredictorV1.from_pretrained(openai_model_name)
+def convert_from_openai_clip(
+    openai_model_name: str, config: Optional[AestheticsPredictorConfig] = None
+) -> AestheticsPredictorV1:
+    config = config or AestheticsPredictorConfig.from_pretrained(openai_model_name)
+    model = AestheticsPredictorV1(config)
+
+    clip_model = CLIPVisionModelWithProjection.from_pretrained(openai_model_name)
+    model.load_state_dict(clip_model.state_dict(), strict=False)
+
     state_dict = torch.hub.load_state_dict_from_url(URLS[openai_model_name])
     model.predictor.load_state_dict(state_dict)
     model.eval()