Create an Instance Segmentation Project#
When your project uses instance segmentation, the uploaded dataset needs to contain the required instance polygons information alongside each image as part of a single datapoint.
First we will create a dataset, and later on, a dummy model that returns a polygon information.
Create dataset#
A convenient approach for creating an instance segmentation dataset is by ensuring that the local format is in COCO-compatible format.
import efemarai as ef
# Create a project
project = ef.Session().create_project(
name="Example Instance Segmentation Project (COCO)",
description="Example project using the COCO dataset format.",
exists_ok=True,
)
dataset = project.create_dataset(
name="Instance Segmentation dataset",
data_url="./data/coco/val2017",
annotations_url="./data/coco/annotations/instances_val2017.json",
stage=ef.DatasetStage.Validation,
format=ef.DatasetFormat.COCO,
)
If your dataset is remote or part of an existing database with custom formats, you can easily upload it to the system by (1) iterating over the dataset and (2) creating a datapoint containing the image and required targets.
In upload_dataset.py add the following script to perform the above steps.
import itertools
import numpy as np
import efemarai as ef
from pycocotools.coco import COCO
from efemarai.formats.dataset_coco import create_mask
from efemarai.fields import create_polygons_from_mask
def upload_dataset():
project = ef.Session().create_project(
name="Example Instance Segmentation Project",
description="Example project with custom datapoint upload.",
exists_ok=True,
)
# Create an empty dataset
dataset = project.create_dataset(
name="Example Custom COCO Dataset",
stage=ef.DatasetStage.Validation,
format=ef.DatasetFormat.Custom,
)
# Define an annotations URL
root = "./data/coco"
# Load your custom dataset
coco = COCO(f"{root}/annotations/instances_val2017.json")
# Create the labels and add them to the dataset
for category in coco.loadCats(coco.getCatIds()):
dataset.add_annotation_class(
id=category["id"],
name=category["name"],
category=category["supercategory"],
)
# Iterate over your dataset
for cocoImg in coco.loadImgs(coco.getImgIds()):
# Instantiate the image with an ef.Image object
image = ef.Image(
file_path=f"{root}/val2017/{cocoImg['file_name']}",
width=cocoImg["width"],
height=cocoImg["height"],
)
# Create a datapoint and add the image as input.
datapoint = ef.Datapoint(
dataset=dataset,
inputs={"image": image},
)
# Get the AnnotationClass from the dataset
annotations = coco.loadAnns(coco.getAnnIds(imgIds=cocoImg["id"]))
for instance_id, cocoAnn in enumerate(annotations):
# Search for the true label object in the dataset by either `name` or `id`.
# Alternatively: label = dataset.get_annotation_class(id=cocoAnn["name"])
label = dataset.get_annotation_class(id=cocoAnn["category_id"])
if isinstance(cocoAnn["segmentation"], dict):
# RLE encoding -> create polygon
mask_img = np.zeros((cocoImg["width"], cocoImg["height"]))
mask_img = ef.formats.dataset_coco.create_mask(mask_img.shape, cocoAnn)
polygons, polygons_area = create_polygons_from_mask(mask_img)
polygons = [list(itertools.chain(*polygon)) for polygon in polygons]
cocoAnn["segmentation"] = polygons
cocoAnn["area"] = polygons_area
vertices = []
for polygon in cocoAnn["segmentation"]:
vertices.append(list(zip(polygon[:-1][::2], polygon[1:][::2])))
# Add a bounding box to the image
datapoint.add_target(
ef.BoundingBox(
xyxy=[
cocoAnn["bbox"][0],
cocoAnn["bbox"][1],
cocoAnn["bbox"][0] + cocoAnn["bbox"][2],
cocoAnn["bbox"][1] + cocoAnn["bbox"][3],
],
label=label,
instance_id=cocoAnn["id"],
ref_field=image,
)
)
datapoint.add_target(
ef.Polygon(
vertices=vertices,
label=label,
instance_id=cocoAnn["id"],
ref_field=image,
)
)
# Upload a single datapoint
datapoint.upload()
# Finalize dataset
dataset.finalize()
if __name__ == "__main__":
upload_dataset()
Run python upload_dataset.py and wait for it to be completed.
After wrapping up any processing, you can confirm the status in the UI and explore the inputs and annotations.
Create a model#
A model that works with bounding boxes dataset will need to return a list of
ef.Polygons objects that will be matched to the ones stored in the dataset.
In a file dummy_model.py save the following code:
import cv2
import numpy as np
import efemarai as ef
import numpy as np
class DummyModel:
"""A DummyModel returning a random polygon"""
def __init__(self, device):
self.device = device # Move model to device
def __call__(self, image):
num_points = 10
points = (np.random.rand(num_points, 2) * [640, 480]).astype(np.float32)
hull = cv2.convexHull(points, clockwise=True)
convex_hull_polygon = hull.squeeze().tolist()
return {
"class_id": np.random.randint(1, 4),
"polygon": [convex_hull_polygon],
"score": np.random.random(),
}
def predict_images(datapoints, model, device):
outputs = []
for datapoint in datapoints:
image = datapoint.get_input("image") # by default, the key name is `image`, this corresponds to the key from the creation dict
image_post_process = image.data / 255 - 0.5 # perform any pre-processing
output = model(image_post_process)
# Here again the label can be referenced by name or class
# label = ef.AnnotationClass(name=output["class_name"])
label = ef.AnnotationClass(id=output["class_id"])
outputs.append(
[
ef.Polygon(
vertices=output["polygon"],
confidence=output["score"], # Confidence of detection
ref_field=image, # Say which image this output refers to
label=label, # And what label it has
),
]
)
# If the model returns a mask, we can use InstanceMask().to_polygon() - e.g.
# mask = np.zeros((480, 640), dtype=np.uint8)
# cv2.polylines(mask, [np.array(convex_hull_polygon)], True, 255, thickness=cv.FILLED)
# ef.InstanceMask(data=mask, ref_field=image, label=label).to_polygon()
return outputs
def load_model(device):
model = DummyModel(device)
return model
def test():
# Load the model
device = "cpu"
model = load_model(device=device)
image_RGB = np.random.randint(0, 256, size=(480, 640, 3), dtype=np.uint8)
ef_datapoint = ef.Datapoint(
dataset=None,
inputs={
"image": ef.Image(data=image_RGB),
},
)
# Pass a list of datapoints to the predictor function
output = predict_images([ef_datapoint], model, device)
print(output)
assert isinstance(output, list)
assert isinstance(output[0][0], ef.Polygon)
if __name__ == "__main__":
test()
If you run it with python dummy_model.py you’ll be able to confirm that the
output of the model is a list of detections per input datapoint.
efemarai.yaml file#
To run the model, you need to have defined the loading and inference
capabilities in the efemarai.yaml file.
Here’s the one corresponding to the dummy model.
project:
name: "Example Instance Segmentation Project"
models:
- name: Dummy Model
description: This is a dummy model to show consuming inputs and outputs
runtime:
image: python:3.10-slim-buster
device: "gpu"
batch:
max_size: 10
load:
entrypoint: dummy_model:load_model
inputs:
- name: device
value: ${model.runtime.device}
output:
name: model
predict:
entrypoint: dummy_model:predict_images
inputs:
- name: datapoints
value: ${datapoints}
- name: model
value: ${model.runtime.load.output.model}
- name: device
value: ${model.runtime.device}
output:
name: predictions
keys:
- masks
Register model#
To register the model, use the CLI to upload it by going into the root of the file directory, next to the efemarai.yaml.
ef model create .
Now you should be able to see the model uploaded and active with this project.