Richard Guo
commited on
Commit
·
de5bc26
1
Parent(s):
ae7183e
fixed indexable field via cast
Browse files- build_map.py +6 -7
build_map.py
CHANGED
|
@@ -143,16 +143,14 @@ def upload_dataset_to_atlas(dataset_dict,
|
|
| 143 |
|
| 144 |
|
| 145 |
# return longest string field from 5 samples
|
|
|
|
| 146 |
if indexed_field is None:
|
| 147 |
longest_length = 0
|
| 148 |
for field in string_fields:
|
| 149 |
length = 0
|
| 150 |
-
for i in range(len(
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
if ex[field]:
|
| 154 |
-
print(ex[field])
|
| 155 |
-
length += len(ex[field].split())
|
| 156 |
if length > longest_length:
|
| 157 |
longest_length = length
|
| 158 |
indexed_field = field
|
|
@@ -262,4 +260,5 @@ if __name__ == "__main__":
|
|
| 262 |
project_name = "huggingface_auto_upload_test-dolly-15k"
|
| 263 |
|
| 264 |
dataset_dict = load_dataset_and_metadata(dataset_name)
|
| 265 |
-
|
|
|
|
|
|
| 143 |
|
| 144 |
|
| 145 |
# return longest string field from 5 samples
|
| 146 |
+
head = dataset_dict["head"]
|
| 147 |
if indexed_field is None:
|
| 148 |
longest_length = 0
|
| 149 |
for field in string_fields:
|
| 150 |
length = 0
|
| 151 |
+
for i in range(len(head)):
|
| 152 |
+
if head[field][i]:
|
| 153 |
+
length += len(str(head[field][i]).split())
|
|
|
|
|
|
|
|
|
|
| 154 |
if length > longest_length:
|
| 155 |
longest_length = length
|
| 156 |
indexed_field = field
|
|
|
|
| 260 |
project_name = "huggingface_auto_upload_test-dolly-15k"
|
| 261 |
|
| 262 |
dataset_dict = load_dataset_and_metadata(dataset_name)
|
| 263 |
+
api_token = "ODdPKqJHYci4Gq4jnCC5-VR0L-rnIdfIy-6djgC4CTPCJ"
|
| 264 |
+
print(upload_dataset_to_atlas(dataset_dict, api_token, project_name=project_name))
|