Importing External Dataset Formats

This tutorial shows how to bring an already-labeled dataset into a Datamint project using datamint.importers, instead of hand-rolling upload_resources() plus a loop of add_box_annotation() calls.

Three formats are supported:

Importer	Format
COCOImporter	COCO JSON (images/annotations/categories)
PascalVOCImporter	Pascal VOC XML (one .xml file per image)
YOLOImporter	YOLO .txt labels (normalized class x_center y_center width height)

All three share the same two-step shape:

• .parse() reads and validates the dataset on disk. Use it to preview image/box counts and class names before uploading anything. Unsupported annotations (e.g. polygons) are counted, not silently dropped.

• .import_to_project(project) reuses the parsed result to upload the images and their box annotations.

This notebook builds tiny synthetic datasets in each format (a couple of generated images) so it runs end-to-end without needing an external download.

Setup

Make sure you’ve run datamint config in a terminal (or set the DATAMINT_API_KEY environment variable) before running this notebook.

import tempfile
from pathlib import Path

from datamint import Api

api = Api()

workdir = Path(tempfile.mkdtemp(prefix="datamint_import_tutorial_"))
print(f"Working directory: {workdir}")

The three sections below each build their own tiny 2-image dataset from this shared layout, so they can be run independently.

from PIL import Image, ImageDraw

IMAGE_SIZE = (128, 128)

# file_name -> (label, (x, y, width, height))
DEMO_BOXES = {
    "image_0.png": ("cat", (10, 10, 40, 30)),
    "image_1.png": ("dog", (20, 15, 35, 25)),
}


def new_demo_image(bbox):
    img = Image.new("RGB", IMAGE_SIZE, color="white")
    x, y, w, h = bbox
    ImageDraw.Draw(img).rectangle([x, y, x + w, y + h], outline="black")
    return img

We’ll import each format into its own project, so the results are easy to tell apart in the web app.

project_coco = api.projects.create(
    "Import Tutorial - COCO", description="datamint.importers tutorial", exists_ok=True
)
project_voc = api.projects.create(
    "Import Tutorial - Pascal VOC", description="datamint.importers tutorial", exists_ok=True
)
project_yolo = api.projects.create(
    "Import Tutorial - YOLO", description="datamint.importers tutorial", exists_ok=True
)

1. Import a COCO dataset

COCOImporter(annotations_file, images_dir=None) reads a single COCO JSON file. images_dir defaults to the annotations file’s parent directory.

First, generate a tiny COCO dataset: two images, one box each.

(``build_coco_dataset`` below is demo-data setup only, not part of the importer API – feel free to skip to the ``COCOImporter`` cell.)

import json


def build_coco_dataset(workdir: Path) -> Path:
    coco_dir = workdir / "coco_dataset"
    coco_dir.mkdir()

    images, annotations = [], []
    for i, (file_name, (label, bbox)) in enumerate(DEMO_BOXES.items()):
        new_demo_image(bbox).save(coco_dir / file_name)

        images.append({"id": i, "file_name": file_name, "width": IMAGE_SIZE[0], "height": IMAGE_SIZE[1]})
        annotations.append({
            "id": i,
            "image_id": i,
            "category_id": 0 if label == "cat" else 1,
            "bbox": list(bbox),
        })

    coco_json = {
        "images": images,
        "annotations": annotations,
        "categories": [{"id": 0, "name": "cat"}, {"id": 1, "name": "dog"}],
    }
    annotations_file = coco_dir / "_annotations.coco.json"
    annotations_file.write_text(json.dumps(coco_json))
    return annotations_file

coco_annotations_file = build_coco_dataset(workdir)

Parse it first to preview what would be uploaded:

from datamint import COCOImporter

coco_importer = COCOImporter(coco_annotations_file)

preview = coco_importer.parse()
print(f"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}")
print(f"missing images: {preview.missing_images}, unsupported annotations: {preview.unsupported_annotations}")

Then upload the images and their box annotations.

result = coco_importer.import_to_project(project_coco, tags=["coco-import-tutorial"])

print(f"Uploaded images: {result.n_images_uploaded}")
print(f"Uploaded boxes: {result.n_boxes_uploaded}")
print(f"Errors: {result.errors}")

2. Import a Pascal VOC dataset

PascalVOCImporter(annotations_dir, images_dir) – unlike COCO, both directories are required explicitly.

(``build_voc_dataset`` below is demo-data setup only, not part of the importer API – feel free to skip to the ``PascalVOCImporter`` cell.)

import xml.etree.ElementTree as ET


def build_voc_dataset(workdir: Path) -> tuple[Path, Path]:
    voc_dir = workdir / "voc_dataset"
    images_dir = voc_dir / "JPEGImages"
    annotations_dir = voc_dir / "Annotations"
    images_dir.mkdir(parents=True)
    annotations_dir.mkdir(parents=True)

    for file_name, (label, bbox) in DEMO_BOXES.items():
        new_demo_image(bbox).save(images_dir / file_name)

        x, y, w, h = bbox
        annotation = ET.Element("annotation")
        ET.SubElement(annotation, "filename").text = file_name
        obj = ET.SubElement(annotation, "object")
        ET.SubElement(obj, "name").text = label
        bndbox = ET.SubElement(obj, "bndbox")
        ET.SubElement(bndbox, "xmin").text = str(x)
        ET.SubElement(bndbox, "ymin").text = str(y)
        ET.SubElement(bndbox, "xmax").text = str(x + w)
        ET.SubElement(bndbox, "ymax").text = str(y + h)
        ET.ElementTree(annotation).write(annotations_dir / f"{Path(file_name).stem}.xml")

    return annotations_dir, images_dir

voc_annotations_dir, voc_images_dir = build_voc_dataset(workdir)

from datamint import PascalVOCImporter

voc_importer = PascalVOCImporter(voc_annotations_dir, voc_images_dir)

preview = voc_importer.parse()
print(f"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}")

result = voc_importer.import_to_project(project_voc, tags=["voc-import-tutorial"])
print(f"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}")

3. Import a YOLO dataset

YOLOImporter(images_dir, labels_dir, class_names=None, data_yaml=None) – YOLO label files are pure numbers with no class name or image filename embedded, so class names must be resolved from one of: an explicit class_names list, an auto-detected data.yaml/data.yml (names: key), or an auto-detected legacy classes.txt. Coordinates are normalized (0..1 of image width/height): class x_center y_center width height.

(``build_yolo_dataset`` below is demo-data setup only, not part of the importer API – feel free to skip to the ``YOLOImporter`` cell.)

def build_yolo_dataset(workdir: Path) -> tuple[Path, Path, list[str]]:
    yolo_dir = workdir / "yolo_dataset"
    images_dir = yolo_dir / "images"
    labels_dir = yolo_dir / "labels"
    images_dir.mkdir(parents=True)
    labels_dir.mkdir(parents=True)

    class_names = ["cat", "dog"]
    img_w, img_h = IMAGE_SIZE

    for file_name, (label, bbox) in DEMO_BOXES.items():
        new_demo_image(bbox).save(images_dir / file_name)

        x, y, w, h = bbox
        cx, cy = (x + w / 2) / img_w, (y + h / 2) / img_h
        nw, nh = w / img_w, h / img_h

        class_id = class_names.index(label)
        label_path = labels_dir / f"{Path(file_name).stem}.txt"
        label_path.write_text(f"{class_id} {cx} {cy} {nw} {nh}\n")

    return images_dir, labels_dir, class_names

yolo_images_dir, yolo_labels_dir, yolo_class_names = build_yolo_dataset(workdir)

from datamint import YOLOImporter

yolo_importer = YOLOImporter(yolo_images_dir, yolo_labels_dir, class_names=yolo_class_names)

preview = yolo_importer.parse()
print(f"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}")

result = yolo_importer.import_to_project(project_yolo, tags=["yolo-import-tutorial"])
print(f"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}")