Usuwanue wszystkich pól o takich samych wartościach w słowniku.

d = {'a':'abc','b':'xxx', ... , 'z':'xxx')}

def transpose(d):
    return {v:k for k,v in d.items()}

transpose(transpose(d))

def remove_duplicate_values(d):
    unique_dict = {}
    for k,v in d.items():
        if v not in unique_dict.values():
            unique_dict[k] = v
    return unique_dict

d = [{'a':'abc'},{'b':'xxx'), ... , {'z':'xxx')]

list({list(x.values())[0]:x for x in d}.values())

                out = []
                for i in d:
                    if str(i['url']).strip() not in str(out):
                                 out.append(i)