<jupyter_start><jupyter_text>Backup labels<jupyter_code>if os.path.isfile(label_dir / "labels.json"):
    os.rename(
        label_dir / "labels.json",
        label_dir / f"labels_{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}.json",
    )
json.dump(
    [omit(l, "_id") for l in vislabels.find()],
    open(label_dir / "labels.json", "w"),
    default=json_util.default,
)<jupyter_output><empty_output><jupyter_text>Labeling<jupyter_code>labelbox_height = 1200
gallery_limit = 50
gallery_height = 1100
# is_wide = False
is_wide = True

all_images = [i for i in imagelabel.find().sort("popularity_score", DESCENDING)]

labelOptions = json.load(open("tmp/labelOptions.json"))
labelOptions = {
    c["category"]: [
        {**o, "subcategory": o["subcategories"][0]} if "subcategories" in o else o
        for o in c["options"]
    ]
    for c in labelOptions
}
labelNamesByTag = {
    f["tag"]: f["name"] for tag, tags in labelOptions.items() for f in tags
}

labelling_states = {
    "filter_states": {"tags": set(), "offset": 0, "image_id": ""},
    "all_images": all_images,
    "images": all_images[:50],
    "current_image_id": "",
    "labelOptions": labelOptions,
}

current_ids = ""
if not os.path.isfile(current_ids_filepath):
    current_ids = PersistentSet()
    current_ids.set_file(current_ids_filepath)
else:
    current_ids = PersistentSet.load_set(current_ids_filepath)


def set_current_image_id(image_id):
    current_ids.persist_remove(list(current_ids)[0])
    current_ids.persist_add(image_id)


def get_current_image_id():
    return list(current_ids)[0]


labelBoxOutput = widgets.Output(layout=Layout(height=f"{labelbox_height+30}px"))
# JSON(labelOptions)
remarks_output = widgets.Output()
remarks_output
def find_image_idx(images, image_id):
    return find_index(images, lambda x: x["image_id"] == image_id)


def find_post(image_id=None, post_id=None):
    if post_id:
        return posts.find_one({"post_id": post_id})
    return posts.find_one(
        {"post_id": imagemeta.find_one({"image_id": image_id})["post_id"]}
    )


def find_label(image_id):
    image = imagelabel.find_one({"image_id": image_id})
    if not image:
        return None
    image_ids = [image_id] + [i for i in image["duplicated_images"]]
    visLabels = [l for l in vislabels.find({"image_id": {"$in": image_ids}})]
    if len(visLabels) > 1:
        print(f"labels warning: more than 1 label {image_ids} {visLabels}")
    return visLabels[0] if visLabels else None


def find_image(image_id):
    if "/" not in image_id:
        image_id = imagemeta.find_one({"short_image_id": image_id}, {"image_id": True})[
            "image_id"
        ]
    image = imagelabel.find_one(
        {"$or": [{"image_id": image_id}, {"duplicated_images": image_id}]}
    )
    if not image:
        print(f"Image not found: {image_id}")
    return image


def find_images_by(tags=[], image_id="", offset=0, limit=50):
    if image_id != "":
        image = find_image(image_id)
        return [image] if image else []

    if len(tags) == 0:
        return labelling_states["all_images"][offset : offset + limit]

    images = []
    for l in vislabels.find({"labels": {"$all": tags}}).limit(limit).skip(offset):
        image = find_image(l["image_id"])
        if image:
            images.append(image)
    #             images.append({**image, 'labels': l['labels']})
    images.sort(key=lambda x: x["popularity_score"])
    images.reverse()
    return images
# filtered_images = []
# for image in all_images:
#     label = find_label(image['image_id'])
#     if label:
#         # flags = [l for l in label['labels'] if 'flag:' in l]
#         # effects = [l for l in label['labels'] if 'effect:' in l]
#         # domains = [l for l in label['labels'] if 'domain:' in l]
#         # if len(flags) > 0 or (len(effects) > 0 and len(domains) > 0):
#         if label['updatedAt'] > datetime.datetime.fromisoformat('2020-04-08T11:20:21'):
#             continue
#         filtered_images.append(image)
# labelling_states['all_images'] = filtered_images
# labelling_states['images'] = filtered_images[:50]

# len(filtered_images)<jupyter_output><empty_output><jupyter_text>Image Box<jupyter_code>def make_image_box(image, width=1000, height=800, layoutArgs={}):
    #     image = widgets.Image(value=open(gallery_dir/image['labelling_path'], 'rb').read(), width=600, layout=Layout(object_position='center center', object_fit='contain'))
    image = widgets.Image(
        value=open(gallery_dir / image["labelling_path"], "rb").read(),
        width=width,
        height=height,
        layout=Layout(object_position="center center", object_fit="contain"),
    )
    boxItems = {"image": image}

    image_box_layout = Layout(justify_content="center", **layoutArgs)
    image_box_layout = Layout(
        height=f"{labelbox_height-500}px", justify_content="center", **layoutArgs
    )
    imageBox = HBox([image], layout=image_box_layout)

    return (imageBox, boxItems)<jupyter_output><empty_output><jupyter_text>Info Box<jupyter_code>def make_link(url, text):
    return widgets.HTML(value=f"<a href='{url}' target='_blank'>{text}</a>")


def make_info_box(image, layoutArgs={}):
    output = widgets.Output()

    links = []
    links.append((find_post(post_id=image["post_id"])["url"], image["post_id"]))
    links += [(find_post(post_id=dp_post['post_id'])["url"], dp_post['post_id']) for dp_post in image["duplicated_posts"]]
    links += [(find_post(dp_image_id)["url"], dp_image_id) for dp_image_id in image["duplicated_images"]]
    links.append((f"{image_url_prefix}/projects/bad-vis-images/{image['labelling_path']}", image["labelling_path"]))

    links = [make_link(*l) for l in links]

    description = widgets.Output(layout=Layout(**layoutArgs))
    with description:
        image_idx_in_all_images = find_image_idx(
            labelling_states["all_images"], image["image_id"]
        )
        len_all_images = len(labelling_states["all_images"])
        print(f"{image_idx_in_all_images} / {len_all_images-1}")
        print(f"popularity_score: {image['popularity_score']}")
        print(f"phash: {image['phash']}")
        print(f"tags: {len(image['tags'])}")
        for t in image["tags"]:
            print(t)

    currentLabelsOutput = widgets.Output(layout=Layout(**layoutArgs))

    outputs = {
        "description": description,
        "currentLabelsOutput": currentLabelsOutput,
        "output": output,
    }

    infoBox = VBox(
        [description, currentLabelsOutput, output, *links]
    )

    return (infoBox, outputs)<jupyter_output><empty_output><jupyter_text>Checkboxes<jupyter_code>def ckb_box_layout(rows, cols, width):
    return Layout(
        flex_flow="column wrap",
        height=f"{rows*32}px",
        width=f"{cols*(width + 5)}px",
        align_content="flex-start",
    )


def make_ckbs(
    title,
    options,
    n_rows=10,
    layoutArgs={},
    on_ckb_change=None,
    grouping=False,
    sorting=True,
    ckb_width=100,
):
    ckb_layout = Layout(width=f"{ckb_width}px")

    layout = Layout(**layoutArgs)
    border_style = layoutArgs.get("border", "3px solid lightblue")
    layout.border = border_style.replace(border_style.split(" ")[-1], "orange")

    form = ""
    subcats = []
    ckbsByName = {}

    def ckb_handler(change):
        ckb = change["owner"]
        if change["new"]:
            ckb.icon = "check"
            ckb.button_style = "success"
        else:
            ckb.icon = ""
            ckb.button_style = ""
        if on_ckb_change:
            on_ckb_change(change)
        if len([v for k, v in ckbsByName.items() if v.value]) > 0:
            form.layout.border = form.layout.border.replace(
                form.layout.border.split(" ")[-1], "lightblue"
            )
        else:
            form.layout.border = form.layout.border.replace(
                form.layout.border.split(" ")[-1], "orange"
            )

    grouped = (
        group_by(options, lambda o: o.get("subcategory", ""))
        if grouping
        else {"": options}
    )
    for subcategory, options in grouped.items():
        ckbs = []
        ordered_options = (
            reversed(sorted(options, key=lambda x: x["count"])) if sorting else options
        )
        for n in ordered_options:
            name = n["tag"]
            ckb = widgets.ToggleButton(
                value=False,
                description=n["name"],
                tooltip=name,
                icon="",
                button_style="",
                disabled=False,
                indent=False,
                layout=ckb_layout,
            )
            ckb.observe(ckb_handler, names="value")
            ckbs.append(ckb)
            ckbsByName[name] = ckb
        subcats.append(
            VBox(
                ckbs,
                layout=ckb_box_layout(
                    rows=n_rows, cols=(-(len(ckbs) // -n_rows)), width=ckb_width
                ),
            )
        )

    form = VBox(
        [
            #         widgets.Label(value=f"{title.capitalize()}"),
            HBox(subcats)
        ],
        layout=layout,
    )

    return (form, ckbsByName)


def make_remarks(n_rows=10, layoutArgs={}):
    form = ""
    remarks = ""

    layout = Layout(**layoutArgs)
    border_style = layoutArgs.get("border", "3px solid lightblue")
    layout.border = border_style.replace(border_style.split(" ")[-1], "orange")

    def remarks_handler(change):
        if remarks.value and remarks.value != "":
            form.layout.border = form.layout.border.replace(
                form.layout.border.split(" ")[-1], "lightblue"
            )
        else:
            form.layout.border = form.layout.border.replace(
                form.layout.border.split(" ")[-1], "orange"
            )

    remarks = widgets.Textarea(
        value="",
        layout=Layout(height=f"{n_rows*30-20}px", width=f"240px"),
        disabled=False,
    )
    remarks.observe(remarks_handler, names="value")
    form = VBox(
        [widgets.Label(value=f"{'remarks'.capitalize()}"), remarks], layout=layout
    )
    return (form, remarks)


def make_ckb_box(options, n_rows=10, layoutArgs={}, on_ckb_change=None):

    formCkbsBox, formCkbsByName = make_ckbs(
        "form",
        options["form"],
        9,
        layoutArgs,
        on_ckb_change,
        grouping=True,
        ckb_width=140,
    )
    layoutCkbsBox, layoutCkbsByName = make_ckbs(
        "layout", options["layout"], 7, layoutArgs, on_ckb_change
    )
    dataCkbsBox, dataCkbsByName = make_ckbs(
        "data", options["data"], n_rows, layoutArgs, on_ckb_change, sorting=False
    )
    domainCkbsBox, domainCkbsByName = make_ckbs(
        "domain", options["domain"], n_rows, layoutArgs, on_ckb_change
    )
    mediaCkbsBox, mediaCkbsByName = make_ckbs(
        "media", options["media"], n_rows, layoutArgs, on_ckb_change
    )
    flagCkbsBox, flagCkbsByName = make_ckbs(
        "flag", options["flag"], n_rows, layoutArgs, on_ckb_change
    )

    faultCkbsBox, faultCkbsByName = make_ckbs(
        "fault",
        options["fault"],
        12,
        layoutArgs,
        on_ckb_change,
        grouping=True,
        sorting=False,
        ckb_width=180,
    )
    effectCkbsBox, effectCkbsByName = make_ckbs(
        "effect", options["effect"], n_rows, layoutArgs, on_ckb_change
    )

    remarksBox, remarks = make_remarks(n_rows, layoutArgs=layoutArgs)

    boxItemsByName = {}
    for d in [
        {"remarks": remarks},
        formCkbsByName,
        faultCkbsByName,
        dataCkbsByName,
        domainCkbsByName,
        layoutCkbsByName,
        effectCkbsByName,
        mediaCkbsByName,
        flagCkbsByName,
    ]:
        for k, v in d.items():
            boxItemsByName[k] = v

    boxesByName = {
        "remarks": remarksBox,
        "form": formCkbsBox,
        "layout": layoutCkbsBox,
        "data": dataCkbsBox,
        "domain": domainCkbsBox,
        "media": mediaCkbsBox,
        "flag": flagCkbsBox,
        "fault": faultCkbsBox,
        "effect": effectCkbsBox,
    }

    return (
        VBox(
            [
                HBox([formCkbsBox, layoutCkbsBox]),
                HBox(
                    [
                        dataCkbsBox,
                        domainCkbsBox,
                        mediaCkbsBox,
                        flagCkbsBox,
                        effectCkbsBox,
                        remarksBox,
                    ]
                ),
                HBox([faultCkbsBox]),
            ]
        ),
        boxItemsByName,
        boxesByName,
    )<jupyter_output><empty_output><jupyter_text>Buttons<jupyter_code>def make_btn_box(on_update=None, on_prev=None, on_next=None):
    btns = []

    if on_prev:
        prevBtn = widgets.Button(
            description="Prev", button_style="warning", layout=Layout(width="60px")
        )

        def go_prev(btn):
            if on_prev:
                on_prev()

        prevBtn.on_click(go_prev)
        btns.append(prevBtn)

        updateAndPrevBtn = widgets.Button(
            description="Save -> Prev", button_style="info"
        )

        def update_and_prev(btn):
            if on_update:
                on_update()
            if on_prev:
                on_prev()

        updateAndPrevBtn.on_click(update_and_prev)
        btns.append(updateAndPrevBtn)

    updateBtn = widgets.Button(description="Save", button_style="success")

    def update(btn):
        if on_update:
            on_update()

    updateBtn.on_click(update)
    btns.append(updateBtn)

    if on_next:
        updateAndNextBtn = widgets.Button(
            description="Save -> Next", button_style="info"
        )

        def update_and_next(btn):
            if on_update:
                on_update()
            if on_next:
                on_next()

        updateAndNextBtn.on_click(update_and_next)
        btns.append(updateAndNextBtn)

        nextBtn = widgets.Button(
            description="Next", button_style="warning", layout=Layout(width="60px")
        )

        def go_next(btn):
            if on_next:
                on_next()

        nextBtn.on_click(go_next)
        btns.append(nextBtn)

    btnItems = {
        "updateBtn": updateBtn,
        "updateAndPrevBtn": updateAndPrevBtn,
        "updateAndNextBtn": updateAndNextBtn,
    }

    return (HBox(btns, layout=Layout(height="34px")), btnItems)<jupyter_output><empty_output><jupyter_text>Label Box<jupyter_code>def make_label_box(image_id, options, next_callback=None, prev_callback=None):
    visImage = imagelabel.find_one({"image_id": image_id})
    if not visImage:
        print(f"Not found: {image_id}")
    vislabel = find_label(image_id)

    labels = set(vislabel["labels"]) if vislabel else set()

    layoutArgs = {"padding": "10px", "margin": "5px", "border": "3px solid lightblue"}

    imageBox, imageBoxItems = make_image_box(
        visImage, height=labelbox_height, layoutArgs=layoutArgs
    )

    infoBox, infoBoxItems = make_info_box(visImage, layoutArgs=layoutArgs)

    invalidLabels = set()

    def show_current_labels():
        currentLabelsOutput = infoBoxItems["currentLabelsOutput"]
        currentLabelsOutput.clear_output()
        with currentLabelsOutput:
            for l in sorted(list(labels)):
                if l in boxItemsByName:
                    print(l)
                else:
                    invalidLabels.add(l)
                    print(colored(l, "red"))

    def on_ckb_change(change):
        ckb = change["owner"]
        labelName = ckb.tooltip
        if change["new"]:
            labels.add(labelName)
        else:
            labels.remove(labelName)
        show_current_labels()

    ckbBox, boxItemsByName, boxesByName = make_ckb_box(
        options, n_rows=7, layoutArgs=layoutArgs, on_ckb_change=on_ckb_change
    )

    remarks = boxItemsByName["remarks"]

    def update_label():
        if remarks.value == "":
            fault_names = [labelNamesByTag[l].capitalize() for l in labels if l.startswith('fault:')]
            if len(fault_names) > 0:
                remarks.value = '\n'.join(fault_names)
        vislabels.find_one_and_update(
            {"image_id": vislabel["image_id"] if vislabel else visImage["image_id"]},
            {
                "$set": {
                    "phash": visImage["phash"],
                    "labels": sorted(list(labels - invalidLabels)),
                    "remarks": remarks.value,
                    "updatedAt": datetime.datetime.utcnow(),
                },
                "$setOnInsert": {"createdAt": datetime.datetime.utcnow()},
            },
            upsert=True,
        )
        with remarks_output:
            if remarks.value:
                print(f"{remarks.value}")
        with infoBoxItems["output"]:
            print(f"{visImage['short_image_id']} updated")
        show_current_labels()

    def on_update():
        update_label()

    def on_prev():
        if prev_callback:
            prev_callback(image_id)

    def on_next():
        if next_callback:
            next_callback(image_id)

    btnBox, btnBoxItems = make_btn_box(
        on_update=on_update, on_prev=on_prev, on_next=on_next
    )

    panelBoxLayout = Layout(
        justify_content="flex-start",
        align_items="flex-start",
        align_content="flex-start",
    )
    panelBox = VBox([ckbBox, btnBox], layout=panelBoxLayout)

    for l in labels:
        if l in boxItemsByName:
            boxItemsByName[l].value = True

    remarks.value = vislabel["remarks"] if vislabel else ""

    with infoBoxItems["output"]:
        print("ready")

    leftBox = VBox([imageBox, infoBox])

    pane_widths = [2, 4, 0] if is_wide else [1, 8, 0]
    return AppLayout(
        left_sidebar=leftBox,
        center=panelBox,
        pane_widths=pane_widths,
        height=f"{labelbox_height+10}px",
    )
def next_id(image_id, prev=False):
    images = (
        labelling_states["all_images"]
        if len(labelling_states["filter_states"]["tags"]) == 0
        else labelling_states["images"]
    )
    idx = find_image_idx(images, image_id)
    new_idx = max(0, idx - 1) if prev else min(len(images) - 1, idx + 1)
    new_image_id = images[new_idx]["image_id"]
    set_current_image_id(new_image_id)
    return new_image_id


def prev_img(output, image_id):
    show_img(next_id(image_id, prev=True), output)


def next_img(output, image_id):
    show_img(next_id(image_id, prev=False), output)

def show_img_file(image_id):
    image = find_image(image_id)
    with open('tmp/currentimage.html', 'w') as out:
        # out.write(f'<html><body><img src="{image_url_prefix}/projects/bad-vis-images/{image["labelling_path"]}"></body></html>')
        out.write(f'''
<html>
<head>
    <style>
        * {{
            margin: 0;
            padding: 0;
        }}
        .imgbox {{
            display: grid;
            height: 100%;
        }}
        .center-fit {{
            max-width: 100%;
            max-height: 100vh;
            margin: auto;
        }}
    </style>
</head>
<body>
<div class="imgbox">
    <img class="center-fit" src="{image_url_prefix}/projects/bad-vis-images/{image["labelling_path"]}">
</div>
</body>
</html>
        ''')
#     with open('tmp/currentpost.html', 'w') as out:
#         out.write(f'''
# <html>
# <body>
# <a href="{image['url']}" target="_blank">{image['url']}</a>
# <iframe id="embededSite" src="{image['url']}" width="100%" height="95%" sandbox="" >
# 	<p>Your browser does not support iframes.</p>
# </iframe>
# </body>
# </html>
#         ''')
    with open('tmp/currentpost.html', 'w') as out:
        out.write(f'''
<html>
<body onload=OpenInNewTab("{image['url']}")>
<script>
function OpenInNewTab(url) {{
  var win = window.open(url, '_blank');
}}
</script>
<a href="{image['url']}" target="_blank">{image['url']}</a>
</body>
</html>
        ''')

def show_img(image_id, output=None):
    output.clear_output()
    show_img_file(image_id)
    with output:
        display(
            make_label_box(
                image_id,
                options=labelOptions,
                next_callback=partial(next_img, output),
                prev_callback=partial(prev_img, output),
            )
        )<jupyter_output><empty_output><jupyter_text>Gallery Box<jupyter_code>def make_filter_box(options, on_filter_change=None, layoutArgs=None):
    layout = Layout(**layoutArgs)

    def ckb_handler(change):
        tag = change["owner"].tag
        if change["new"]:
            labelling_states["filter_states"]["tags"].add(tag)
        else:
            labelling_states["filter_states"]["tags"].remove(tag)
        offset_textbox.value = 0
        if on_filter_change:
            on_filter_change(labelling_states["filter_states"])

    def image_id_textbox_handler(change):
        labelling_states["filter_states"]["image_id"] = change["new"]
        on_filter_change(labelling_states["filter_states"])

    def offset_textbox_handler(change):
        labelling_states["filter_states"]["offset"] = change["new"]
        on_filter_change(labelling_states["filter_states"])

    def clear_handler(change):
        for ckb in ckbs:
            ckb.value = False
        offset_textbox.value = 0
        image_id_textbox.value = ""

    panel_widgets = []

    image_id_textbox = widgets.Text(
        value="",
        placeholder="1yb04c_0",
        description="Image ID:",
        disabled=False,
        layout={"width": "180px"},
    )
    image_id_textbox.observe(image_id_textbox_handler, names="value")
    panel_widgets.append(image_id_textbox)

    offset_textbox = widgets.BoundedIntText(
        value=labelling_states["filter_states"]["offset"],
        min=0,
        max=len(labelling_states["all_images"]),
        step=gallery_limit,
        description="Offset",
        disabled=False,
        layout={"width": "180px"},
    )
    offset_textbox.observe(offset_textbox_handler, names="value")
    panel_widgets.append(offset_textbox)

    #     clear_btn = widgets.Button(description='clear filter', tooltip='Clear', layout={'width':'180px'}) # button widgets are bugged, zero height and invisible
    #     clear_btn.on_click(clear_handler)
    clear_btn = widgets.Checkbox(
        value=False, description="Clear Filter", disabled=False
    )
    clear_btn.observe(clear_handler, names="value")
    panel_widgets.append(clear_btn)

    ckbs = []
    for cat, opts in options.items():
        for subcat, options in group_by(
            opts, lambda o: o.get("subcategory", "")
        ).items():
            panel_widgets.append(
                widgets.widgets.HTML(
                    value=f"<b>{cat.upper()}</b> {subcat.capitalize()}"
                )
            )
            for o in options:
                ckb = widgets.Checkbox(
                    value=False,
                    description=f"{o['name']} ({o['count']})",
                    disabled=False,
                    indent=False,
                )
                ckb.tag = o["tag"]
                ckb.observe(ckb_handler, names="value")
                ckbs.append(ckb)
                panel_widgets.append(ckb)

    return VBox(panel_widgets, layout=layout)


def make_gallery_box(images, on_select=None, layoutArgs=None):
    def click_handler(btn):
        image_id = btn.tooltip
        if on_select:
            on_select(image_id)

    image_boxes = []
    for i in images:
        image_box = widgets.Image(
            value=open(f"../{i['labelling_thumbnail_path']}", "rb").read(),
            width=140,
            height=140,
            layout=Layout(width="140px", height="140px"),
        )
        btn = widgets.Button(
            description=f"{i['image_name']} ({i['popularity_score']})",
            tooltip=i["image_id"],
        )
        btn.on_click(click_handler)
        image_boxes.append(VBox([image_box, btn]))

    return Box(image_boxes, layout=Layout(display="flex", flex_flow="row wrap"))


def make_gallery(options, select_callback=None):
    layoutArgs = {"padding": "10px", "margin": "5px", "border": "3px solid lightblue"}

    def select_handler(image_id):
        show_img(image_id, labelBoxOutput)

    labelling_states["images"] = find_images_by(
        limit=gallery_limit, offset=labelling_states["filter_states"]["offset"]
    )

    galleryOutput = widgets.Output(
        layout=Layout(**layoutArgs, height=f"{gallery_height-20}px")
    )
    with galleryOutput:
        display(
            make_gallery_box(
                labelling_states["images"],
                on_select=select_handler,
                layoutArgs=layoutArgs,
            )
        )

    def filter_handler(filter_states):
        labelling_states["images"] = find_images_by(
            tags=list(filter_states["tags"]),
            image_id=filter_states["image_id"],
            offset=filter_states["offset"],
            limit=gallery_limit,
        )
        galleryOutput.clear_output()
        with galleryOutput:
            display(
                make_gallery_box(
                    labelling_states["images"],
                    on_select=select_handler,
                    layoutArgs=layoutArgs,
                )
            )

    filterBox = make_filter_box(
        options, on_filter_change=filter_handler, layoutArgs=layoutArgs
    )

    pane_widths = [1, 5, 1]
    return AppLayout(
        left_sidebar=filterBox,
        center=galleryOutput,
        pane_widths=pane_widths,
        height=f"{gallery_height}px",
    )<jupyter_output><empty_output><jupyter_text>GUI<jupyter_code>make_gallery(labelling_states["labelOptions"])
labelBoxOutput
show_img(get_current_image_id(), labelBoxOutput)
currentimage_url = f"{image_url_prefix}/projects/bad-vis-images/notebooks/tmp/currentimage.html"
make_link(currentimage_url, f"current labeling image browser sync link: {currentimage_url}")
currentpost_url = f"{image_url_prefix}/projects/bad-vis-images/notebooks/tmp/currentpost.html"
make_link(currentpost_url, f"current labeling image browser sync link: {currentpost_url}")<jupyter_output><empty_output><jupyter_text>Maintenance<jupyter_code># for label in vislabels.find():
#     visimage = imagelabel.find_one({'image_id': label['image_id']})
#     if not visimage:
# #         print(f"missing {label['image_id']}")
#         duplicated = [i for i in imagelabel.find({'duplicated_images': label['image_id']})]
#         if len(duplicated) > 1:
#             pass
# #             print(f"found more than one duplicated {label['image_id']} {duplicated}")
#         if not duplicated:
# #             pass
#             print(f"missing in duplicated either {label['image_id']}")
#             print(json.dumps(omit(label, '_id'), default=datetime.datetime.isoformat))
#         else:
#             duplicated = duplicated[0]
# #             print(f"found in duplicated {label['image_id']} {duplicated['image_id']}")
# for label in vislabels.find():
#     if 'tumblr' in label['image_id']:
#         continue

#     visimage = imagelabel.find_one({'image_id': label['image_id']})
#     if not visimage:
#         visimage = imagelabel.find_one({'duplicated_images': label['image_id']})

#     if not visimage:
#         print(f"missing {label['image_id']}")
#     else:
#         label['image_id'] = visimage['image_id']
#         label['phash'] = visimage['phash']
#         vislabels.update_one({'_id': label['_id']}, {'$set': {'image_id': label['image_id'], 'phash': label['phash']}})
# # replace labels
# current_label = 'fault:percentage'
# new_label = ''

# for l in vislabels.find():
#     if len([l for l in l['labels'] if current_label == l]) > 0:
#         ori_length = len(l['labels'])
#         print(len(l['labels']), l['_id'], l['image_id'], l['labels'])

#         l['labels'] = [l for l in l['labels'] if current_label != l]
# #         if new_label not in l['labels']:
# #             l['labels'].append(new_label)

#         new_length = len(l['labels'])
#         print(new_length - ori_length, new_label in l['labels'], len(l['labels']), l['labels'])

# #         vislabels.update_one({'_id': l['_id']}, {'$set': {'labels': l['labels']}})
# #     break<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13616689136450721, 'fraction_numerical': 0.0062797133328602855, 'mean_word_length': 2.588415022278803, 'pattern_counts': {'":': 60, '<': 79, '<?xml version=': 0, '>': 90, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '214766', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 9293, 'n_tokens_neox': 8996, 'n_words': 1556, 'path': 'notebooks/images_labeling.ipynb', 'repo_name': 'leoyuholo/bad-vis-images', 'size': '28186'}