skip to Main Content

I have two input JSON files:

The files are way too long to post here, but they follow the form below:

data_1 = {
    "accessibility-service": {
        "docmagic-prod-torrance": {
            "images": [
                "harbor.docmagic.com/docsys/accessibility-service:1.0.10-86df397-121321102854",
                "harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "accessibilitycrawford-service": {
        "docmagic-prod-torrance": {
            "images": [
                "harbor.docmagic.com/docsys/accessibilitycrawford-service:master-ebc95d6-20211006145840",
                "harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "accounting-service": {
        "docmagic-prod-torrance": {
            "images": [
                "harbor.docmagic.com/docsys/accounting-service:4.1.0-9a8b850-111621144747",
                "harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "authentication-service": {
        "docmagic-prod-torrance": {
            "images": [
                "harbor.docmagic.com/docsys/authentication-service:3.2.0-3da5692-020222162357",
                "harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "autoprep-api": {
        "docmagic-prod-torrance": {
            "images": [
                "harbor.docmagic.com/docsys/autoprep-api:3.16.4-4830179-030122110030",
                "harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "autoprep-ui": {
        "docmagic-prod-torrance": {
            "images": [
                "harbor.docmagic.com/devops/config-init:master-91ea56d-062720021322",
                "harbor.docmagic.com/docsys/autoprep-ui:1.23.3-d8fc26c-020922101408"
            ]
        }
    },
    "autoprepeditor-ui": {
        "docmagic-prod-torrance": {
            "images": []
        }
    },
    "billingevent-service": {
        "docmagic-prod-torrance": {
            "images": []
        }
    },
    "cascustomer-service": {
        "docmagic-prod-torrance": {
            "images": [
                "harbor.docmagic.com/library/busybox:1.32.0",
                "harbor.docmagic.com/library/redis:6.0.9-alpine"
            ]
        }
    }
}

The above is being compared with the below:

data_2 = {
    "accessibility-service": {
        "docmagic-prod-austin": {
            "images": [
                "dr-harbor.docmagic.com/docsys/accessibility-service:1.0.10-86df397-121321102854",
                "harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "accessibilitycrawford-service": {
        "docmagic-prod-austin": {
            "images": [
                "dr-harbor.docmagic.com/docsys/accessibilitycrawford-service:master-ebc95d6-20211006145840",
                "harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "accounting-service": {
        "docmagic-prod-austin": {
            "images": [
                "dr-harbor.docmagic.com/docsys/accounting-service:4.1.0-9a8b850-111621144747",
                "harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "authentication-service": {
        "docmagic-prod-austin": {
            "images": [
                "dr-harbor.docmagic.com/docsys/authentication-service:3.2.0-3da5692-020222162357",
                "dr-harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "autoprep-api": {
        "docmagic-prod-austin": {
            "images": [
                "dr-harbor.docmagic.com/docsys/autoprep-api:3.16.4-4830179-030122110030",
                "dr-harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "autoprep-ui": {
        "docmagic-prod-austin": {
            "images": [
                "dr-harbor.docmagic.com/devops/config-init:master-91ea56d-062720021322",
                "dr-harbor.docmagic.com/docsys/autoprep-ui:1.23.3-d8fc26c-020922101408"
            ]
        }
    },
    "autoprepeditor-ui": {
        "docmagic-prod-austin": {
            "images": []
        }
    },
    "billingevent-service": {
        "docmagic-prod-austin": {
            "images": []
        }
    },
    "cascustomer-service": {
        "docmagic-prod-austin": {
            "images": [
                "dr-harbor.docmagic.com/docsys/cascustomer-service:4.8.0-SNAPSHOT-20210623172337-062321102337",
                "dr-harbor.docmagic.com/library/busybox:1.32.0",
                "dr-harbor.docmagic.com/library/dm-nginx:latest",
                "dr-harbor.docmagic.com/library/redis:6.0.9-alpine"
            ]
        }
    },
    "cascustomer-service-v6": {
        "docmagic-prod-austin": {
            "images": [
                "dr-harbor.docmagic.com/docsys/cascustomer-service-v6:6.5.1-dacdb66-022822144836",
                "dr-harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    },
    "casfederation-service": {
        "docmagic-prod-austin": {
            "images": [
                "dr-harbor.docmagic.com/docsys/casfederation-service:1.3.5-3a6664f-022322174350",
                "dr-harbor.docmagic.com/library/dm-nginx:latest"
            ]
        }
    }
}

The code I have so far, returns identical strings. It should only return strings that are either nonexistent in the comparative, or different after the colon in each string.

#!/usr/bin/python3
import json,urllib.request,difflib,datetime,smtplib,os,email.message,csv,sys,base64
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from datetime import date
from email.message import EmailMessage

diff = difflib.Differ()

def createEnv(name, url):
    env = dict(name=name, url=url)
    return env

def create(torranceEnvName, torranceEnvUrl, austinName, austinUrl):
    torranceEnv = createEnv(torranceEnvName, torranceEnvUrl)
    austinEnv = createEnv(austinName, austinUrl)
    env = dict(torranceEnv=torranceEnv, austinEnv=austinEnv)
    return env



def normalizeAustinValues(imageTag):
    austinHost = 'dr-harbor.docmagic.com'
    torranceHost = 'harbor.docmagic.com'
    if (austinHost in imageTag):
        imageTag = imageTag.replace(austinHost, torranceHost)
    return imageTag

def compareEnvironmentTags(torranceEnv, austinEnv, report_file):

    data_1 = urllib.request.urlopen(torranceEnv['url']).read()
    data_2 = urllib.request.urlopen(austinEnv['url']).read()

    torrance_dict = json.loads(data_1)
    austin_dict = json.loads(data_2)

    missing_in_torrance = []
    missing_in_austin = []
    check_for_diffs = []

    for i in austin_dict:
        if (i not in torrance_dict):
            missing_in_torrance.append(i)

    for i in torrance_dict:
        if (i not in austin_dict):
            missing_in_austin.append(i)
        else:
            check_for_diffs.append(i)

    for i in missing_in_torrance:
        print('tmissing in torrance: ', i, file=report_file)
    for i in missing_in_austin:
        print('tmissing in austin  : ', i, file=report_file)

    print(file=report_file)
    for i in check_for_diffs:
        
        torrance_images = torrance_dict[i][torranceEnv['name']]['images']
        austin_images = austin_dict[i][austinEnv['name']]['images']

        torrance_images = list(torrance_images)
        austin_images = list(austin_images)
        normalized_austin_images = list(map(normalizeAustinValues, austin_images))
        
        torrance_images.sort()
        austin_images.sort()
        normalized_austin_images.sort()

        result = diff.compare(torrance_images, normalized_austin_images)
        result = [line for line in result if line.startswith(("- ", "+ "))]
        result = ''.join(result)

        if (result): 
            print('t', i, file=report_file)
            
            if (len(torrance_images) != len(austin_images)):
                print('tt torrance', file=report_file)
                for image in torrance_images:
                    print('ttt', image, file=report_file)
                print('tt austin', file=report_file)
                for image in austin_images:
                    print('ttt', image, file=report_file)
            else:
                for idx,image in enumerate(torrance_images):
                    if (image != normalized_austin_images[idx]):
                        print('tt torrance', file=report_file)
                        print('ttt ', image, file=report_file)
                        print('tt austin', file=report_file)
                        print('ttt ', austin_images[idx], file=report_file)

    return #compareEnvironmenTags

### execution block

environments = []
environments.append(create("docmagic-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/docmagic-prod-torrance", "docmagic-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/docmagic-prod-austin"))
environments.append(create("docmagic-dev-torrance", "https://gitops-service.docmagic.com/argocd/compare/docmagic-dev-torrance", "docmagic-dev-austin", "https://gitops-service.docmagic.com/argocd/compare/docmagic-dev-austin"))
environments.append(create("uwm-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/uwm-prod-torrance", "uwm-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/uwm-prod-austin"))
environments.append(create("eps-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/eps-prod-torrance", "eps-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/eps-prod-austin"))
environments.append(create("jpmc-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/jpmc-prod-torrance", "jpmc-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/jpmc-prod-austin"))
environments.append(create("wf-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/wf-prod-torrance", "wf-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/wf-prod-austin"))

with open('report.txt', 'w') as report_file:
    for env in environments:
        print(env['torranceEnv']['name'], ' vs ', env['austinEnv']['name'], file=report_file)
        compareEnvironmentTags(env['torranceEnv'], env['austinEnv'], report_file)
        print(file=report_file)

For instance, I should not be getting back identical strings like the image below:

enter image description here

I believe what needs to be edited in my script is this section here:

            if (len(torrance_images) != len(austin_images)):
                print('tt torrance', file=report_file)
                for image in torrance_images:
                    print('ttt', image, file=report_file)
                print('tt austin', file=report_file)
                for image in austin_images:
                    print('ttt', image, file=report_file)
            else:

After updating my script, with your changes @JonSG, I can get it to print and send email, but it doesn’t do each vertical+environment one at a time based on the execution block below like the original script did. I couldn’t figure out how to get it to print to file, so I ran the script from within the script and redirected it’s output to a file and piped that into the body of an email.

#!/usr/bin/python3
import json,urllib.request,difflib,datetime,smtplib,os,email.message,csv,sys,base64,stat
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from datetime import date
from email.message import EmailMessage
from subprocess import call

with open ('diff-report.py', 'w') as rsh:
    rsh.write('''
#!/usr/bin/python3
import json,urllib.request,difflib,datetime,smtplib,os,email.message,csv,sys,base64,stat
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from datetime import date
from email.message import EmailMessage
from subprocess import call

diff = difflib.Differ()

def createEnv(name, url):
    env = dict(name=name, url=url)
    return env

def create(torranceEnvName, torranceEnvUrl, austinName, austinUrl):
    torranceEnv = createEnv(torranceEnvName, torranceEnvUrl)
    austinEnv = createEnv(austinName, austinUrl)
    env = dict(torranceEnv=torranceEnv, austinEnv=austinEnv)
    return env

def normalizeAustinValues(imageTag):
    austinHost = 'dr-harbor.docmagic.com'
    torranceHost = 'harbor.docmagic.com'
    if (austinHost in imageTag):
        imageTag = imageTag.replace(austinHost, torranceHost)
    return imageTag

def compareEnvironmentTags(torranceEnv, austinEnv, report_file):

    data_1 = urllib.request.urlopen(torranceEnv['url']).read()
    data_2 = urllib.request.urlopen(austinEnv['url']).read()

    torrance_dict = json.loads(data_1)
    austin_dict = json.loads(data_2)

    for torrance_service_key, torrance_service_value in torrance_dict.items():
        torrance_service_images = list(torrance_service_value.values())[0]["images"]

        austin_service_value = austin_dict.get(torrance_service_key, {})
        austin_service_images = list(austin_service_value.values())[0]["images"] if list(austin_service_value.values()) else []
        austin_service_images = [i.replace("dr-harbor.docmagic.com", "harbor.docmagic.com") for i in austin_service_images]

        torrance_not_in_austin = set(torrance_service_images).difference(austin_service_images)
        austin_not_in_torrance = set(austin_service_images).difference(torrance_service_images)
                                
        if torrance_not_in_austin or austin_not_in_torrance:
            print(f"{torrance_service_key}:")
            print("tonly in torrance:")
            for image in torrance_not_in_austin:
                print(f"tt{image}")

            print("tonly in austin:")
            for image in austin_not_in_torrance:
                print(f"tt{image}")
                                                                                                                                                    
### execution block

environments = []
environments.append(create("docmagic-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/docmagic-prod-torrance", "docmagic-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/docmagic-prod-austin"))
environments.append(create("docmagic-dev-torrance", "https://gitops-service.docmagic.com/argocd/compare/docmagic-dev-torrance", "docmagic-dev-austin", "https://gitops-service.docmagic.com/argocd/compare/docmagic-dev-austin"))
environments.append(create("uwm-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/uwm-prod-torrance", "uwm-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/uwm-prod-austin"))
environments.append(create("eps-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/eps-prod-torrance", "eps-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/eps-prod-austin"))
environments.append(create("jpmc-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/jpmc-prod-torrance", "jpmc-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/jpmc-prod-austin"))
environments.append(create("wf-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/wf-prod-torrance", "wf-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/wf-prod-austin"))

with open('report.txt', 'w') as report_file:
    for env in environments:
        print(env['torranceEnv']['name'], ' vs ', env['austinEnv']['name'], file=report_file)
        compareEnvironmentTags(env['torranceEnv'], env['austinEnv'], report_file)
        print(file=report_file)
''')

st = os.stat('diff-report.py')
os.chmod('diff-report.py', st.st_mode | stat.S_IEXEC)

rc = call("./diff-report.py >> report.txt", shell=True)

# Sender and recipient
me = '[email protected]'
you = '[email protected]'      
# Open the plain text file whose name is in textfile for reading.
textfile = 'report.txt'
with open(textfile) as fp:
    # Create a text/plain message
    msg = EmailMessage()
    msg.set_content(fp.read())

# me == the sender's email address
# you == the recipient's email address
date = datetime.datetime.now().strftime('%m-%d-%Y')
msg['Subject'] = f'Image Diff Report %s' % date
msg['From'] = me
msg['To'] = you

# Send the message via our own SMTP server.
s = smtplib.SMTP('smtp-xxxxxxx.xxxxxxxx.com:25')
s.send_message(msg)
s.quit()
os.remove("report.txt")
os.remove("diff-report.py")

But what I get is, the code only runs through one comparison, and it actually repeats itself on that comparison. It should rather put each comparison after this line prints in the for loop:

       print(env['torranceEnv']['name'], ' vs ', env['austinEnv']['name'], file=report_file)

This is what prints out now:

docmagic-prod-torrance  vs  docmagic-prod-austin

docmagic-dev-torrance  vs  docmagic-dev-austin

uwm-prod-torrance  vs  uwm-prod-austin

eps-prod-torrance  vs  eps-prod-austin

jpmc-prod-torrance  vs  jpmc-prod-austin

wf-prod-torrance  vs  wf-prod-austin

cascustomer-service:
    only in torrance:
    only in austin:
        harbor.docmagic.com/docsys/cascustomer-service:4.8.0-SNAPSHOT-20210623172337-062321102337
        harbor.docmagic.com/library/dm-nginx:latest
cleanup-providerservices-mysql:
    only in torrance:
        harbor.docmagic.com/devops/providerservices-mysql-cronjob:master-baddfe9-101121133600
    only in austin:
customer-expiration-notification-cli:
    only in torrance:
        harbor.docmagic.com/docsys/customer-expiration-notification-cli:1.2.2-c80493c-011822135312
    only in austin:
dmdirect-api:
    only in torrance:
        harbor.docmagic.com/docsys/dsitemplates-init:app1-030722181252
    only in austin:
        harbor.docmagic.com/docsys/dsitemplates-init:app1-030722184922
eeligibility-service-cli:
    only in torrance:
        harbor.docmagic.com/docsys/eeligibility-service-cli:master-251852f-042721181924
    only in austin:
reporting-service:
    only in torrance:
        harbor.docmagic.com/docsys/dsiconfig-init:fm1-051021191732
    only in austin:
        harbor.docmagic.com/docsys/dsiconfig-init:fm1-051021200915
autoprepeditor-ui:
    only in torrance:
        harbor.docmagic.com/devops/config-init:master-91ea56d-062720021322
        harbor.docmagic.com/docsys/autoprepeditor-ui:develop-7958f2c-022422105824
    only in austin:
cascustomer-service:
    only in torrance:
        harbor.docmagic.com/docsys/cascustomer-service:1.1.6-SNAPSHOT-20211221224307-122121144307
    only in austin:
        harbor.docmagic.com/docsys/cascustomer-service:4.8.0-SNAPSHOT-20210623172337-062321102337
customer-expiration-notification-cli:
    only in torrance:
        harbor.docmagic.com/docsys/customer-expiration-notification-cli:develop-54ec2ab-011822135341
    only in austin:
distill-api:
    only in torrance:
        harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-030722132307
        harbor.docmagic.com/docsys/dsimaps-init:dev-app1-022822115534
    only in austin:
        harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-121021143604
        harbor.docmagic.com/docsys/dsimaps-init:dev-app1-120321163129
dmdirect-api:
    only in torrance:
        harbor.docmagic.com/docsys/dsitemplates-init:dev-app1-021522140300
        harbor.docmagic.com/docsys/dsimaps-init:dev-app1-022822115534
        harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-031422114821
    only in austin:
        harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-102221083221
        harbor.docmagic.com/docsys/dsimaps-init:prod-102120174004
        harbor.docmagic.com/docsys/dsitemplates-init:prod-090820124603
eeligibility-service-cli:
    only in torrance:
        harbor.docmagic.com/docsys/eeligibility-service-cli:develop-09ce3c8-101921110938
    only in austin:
evaultstorage-service:
    only in torrance:
        harbor.docmagic.com/docsys/keystore-init:20210810
        harbor.docmagic.com/docsys/evaultstorage-service:develop-b1d5c18-030322112731
        harbor.docmagic.com/library/dm-nginx:latest
    only in austin:
fillmagic-service:
    only in torrance:
        harbor.docmagic.com/docsys/dsiconfig-init:dev-fm1-030922095928
    only in austin:
        harbor.docmagic.com/docsys/dsiconfig-init:dev-fm1-101321163332
formanalyzer-cli:
    only in torrance:
        harbor.docmagic.com/docsys/formanalyzer-cli:develop-859ddaa-031022090930
        harbor.docmagic.com/docsys/formanalyzer-cli:develop-859ddaa-031122121155
    only in austin:
fulfillment-cli:
    only in torrance:
        harbor.docmagic.com/docsys/fulfillment-cli:develop-77eb2ec-121321112830
        harbor.docmagic.com/docsys/fulfillment-cli:develop-e674f67-012522125827
    only in austin:
identity-api:
    only in torrance:
        harbor.docmagic.com/docsys/identity-api:develop-9b96e3d-030922095254
    only in austin:
        harbor.docmagic.com/docsys/identity-api:master-366d404-051021133325
providerservices-api:
    only in torrance:
        harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-030922102038
    only in austin:
        harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-042121161803
reporting-service:
    only in torrance:
        harbor.docmagic.com/docsys/dsitemplates-init:dev-fm1-011422173814
        harbor.docmagic.com/docsys/dsiconfig-init:dev-fm1-050521142841
    only in austin:
        harbor.docmagic.com/docsys/dsiconfig-init:app1-042921164657
        harbor.docmagic.com/docsys/dsitemplates-init:prod-090120110639
cleanup-providerservices-mysql:
    only in torrance:
        harbor.docmagic.com/devops/providerservices-mysql-cronjob:master-baddfe9-101121133600
    only in austin:
customer-expiration-notification-cli:
    only in torrance:
        harbor.docmagic.com/docsys/customer-expiration-notification-cli:1.2.2-c80493c-011822135312
    only in austin:
eeligibility-service-cli:
    only in torrance:
        harbor.docmagic.com/docsys/eeligibility-service-cli:master-251852f-042721181924
    only in austin:
cleanup-providerservices-mysql:
    only in torrance:
        harbor.docmagic.com/devops/providerservices-mysql-cronjob:master-baddfe9-101121133600
    only in austin:
customer-expiration-notification-cli:
    only in torrance:
        harbor.docmagic.com/docsys/customer-expiration-notification-cli:develop-54ec2ab-011822135341
    only in austin:
eeligibility-service-cli:
    only in torrance:
        harbor.docmagic.com/docsys/eeligibility-service-cli:master-251852f-042721181924
    only in austin:
cascustomer-service:
    only in torrance:
        harbor.docmagic.com/docsys/cascustomer-service:4.8.0-SNAPSHOT-20210623172337-062321102337
        harbor.docmagic.com/library/dm-nginx:latest
    only in austin:
reporting-service:
    only in torrance:
        harbor.docmagic.com/docsys/dsitemplates-init:jpmc-fm1-022422155417
        harbor.docmagic.com/docsys/dsiconfig-init:jpmc-fm1-020822181951
    only in austin:
        harbor.docmagic.com/docsys/dsitemplates-init:jpmc-fm1-020822182115
        harbor.docmagic.com/docsys/dsiconfig-init:fm1-051021191732
esignpackage-service:
    only in torrance:
        harbor.docmagic.com/docsys/esignpackage-service:1.7.5-57b7c00-111521160457
    only in austin:
        harbor.docmagic.com/docsys/esignpackage-service:1.8.1-b6c674d-120121150952
nmlsupdate-cli:
    only in torrance:
        harbor.docmagic.com/docsys/nmlsupdate-cli:1.0-d22848d-100521134007
    only in austin:

The correct format of the output should look exactly like below, with the exception of the fact that it has some exact matches, there should be no exact matches.

docmagic-prod-torrance  vs  docmagic-prod-austin
    missing in austin  :  cleanup-providerservices-mysql
    missing in austin  :  customer-expiration-notification-cli
    missing in austin  :  eeligibility-service-cli
    missing in austin  :  evaultstorage-service

     cascustomer-service
         torrance
             harbor.docmagic.com/library/busybox:1.32.0
             harbor.docmagic.com/library/redis:6.0.9-alpine
         austin
             dr-harbor.docmagic.com/docsys/cascustomer-service:4.8.0-SNAPSHOT-20210623172337-062321102337
             dr-harbor.docmagic.com/library/busybox:1.32.0
             dr-harbor.docmagic.com/library/dm-nginx:latest
             dr-harbor.docmagic.com/library/redis:6.0.9-alpine
     dmdirect-api
         torrance
              harbor.docmagic.com/docsys/dsitemplates-init:app1-030722181252
         austin
              dr-harbor.docmagic.com/docsys/dsitemplates-init:app1-030722184922
     reporting-service
         torrance
              harbor.docmagic.com/docsys/dsiconfig-init:fm1-051021191732
         austin
              dr-harbor.docmagic.com/docsys/dsiconfig-init:fm1-051021200915

docmagic-dev-torrance  vs  docmagic-dev-austin
    missing in torrance:  cleanup-dsivar
    missing in austin  :  customer-expiration-notification-cli
    missing in austin  :  eeligibility-service-cli
    missing in austin  :  evaultstorage-service
    missing in austin  :  fulfillment-cli

     autoprepeditor-ui
         torrance
             harbor.docmagic.com/devops/config-init:master-91ea56d-062720021322
             harbor.docmagic.com/docsys/autoprepeditor-ui:develop-7958f2c-022422105824
         austin
     cascustomer-service
         torrance
              harbor.docmagic.com/docsys/cascustomer-service:1.1.6-SNAPSHOT-20211221224307-122121144307
         austin
              dr-harbor.docmagic.com/docsys/cascustomer-service:4.8.0-SNAPSHOT-20210623172337-062321102337
     distill-api
         torrance
              harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-030722132307
         austin
              dr-harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-121021143604
         torrance
              harbor.docmagic.com/docsys/dsimaps-init:dev-app1-022822115534
         austin
              dr-harbor.docmagic.com/docsys/dsimaps-init:dev-app1-120321163129
     dmdirect-api
         torrance
              harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-030822140304
         austin
              dr-harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-102221083221
         torrance
              harbor.docmagic.com/docsys/dsimaps-init:dev-app1-022822115534
         austin
              dr-harbor.docmagic.com/docsys/dsimaps-init:prod-102120174004
         torrance
              harbor.docmagic.com/docsys/dsitemplates-init:dev-app1-021522140300
         austin
              dr-harbor.docmagic.com/docsys/dsitemplates-init:prod-090820124603
     fillmagic-service
         torrance
              harbor.docmagic.com/docsys/dsiconfig-init:dev-fm1-030922095928
         austin
              dr-harbor.docmagic.com/docsys/dsiconfig-init:dev-fm1-101321163332
     formanalyzer-cli
         torrance
             harbor.docmagic.com/docsys/formanalyzer-cli:develop-859ddaa-031022090930
             harbor.docmagic.com/docsys/formanalyzer-cli:develop-859ddaa-031122121155
         austin
     identity-api
         torrance
              harbor.docmagic.com/docsys/identity-api:develop-9b96e3d-030922095254
         austin
              dr-harbor.docmagic.com/docsys/identity-api:master-366d404-051021133325
     providerservices-api
         torrance
              harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-030922102038
         austin
              dr-harbor.docmagic.com/docsys/dsiconfig-init:dev-app1-042121161803
     reporting-service
         torrance
              harbor.docmagic.com/docsys/dsiconfig-init:dev-fm1-050521142841
         austin
              dr-harbor.docmagic.com/docsys/dsiconfig-init:app1-042921164657
         torrance
              harbor.docmagic.com/docsys/dsitemplates-init:dev-fm1-011422173814
         austin
              dr-harbor.docmagic.com/docsys/dsitemplates-init:prod-090120110639

uwm-prod-torrance  vs  uwm-prod-austin
    missing in austin  :  cleanup-providerservices-mysql
    missing in austin  :  customer-expiration-notification-cli
    missing in austin  :  eeligibility-service-cli


eps-prod-torrance  vs  eps-prod-austin
    missing in austin  :  cleanup-providerservices-mysql
    missing in austin  :  customer-expiration-notification-cli
    missing in austin  :  eeligibility-service-cli


jpmc-prod-torrance  vs  jpmc-prod-austin
    missing in torrance:  formmetadata-service

     cascustomer-service
         torrance
             harbor.docmagic.com/docsys/cascustomer-service:4.8.0-SNAPSHOT-20210623172337-062321102337
             harbor.docmagic.com/library/busybox:1.32.0
             harbor.docmagic.com/library/dm-nginx:latest
             harbor.docmagic.com/library/redis:6.0.9-alpine
         austin
             dr-harbor.docmagic.com/library/busybox:1.32.0
             dr-harbor.docmagic.com/library/redis:6.0.9-alpine
     eeligibility-service-cli
         torrance
             harbor.docmagic.com/docsys/eeligibility-service-cli:master-251852f-042721181924
         austin
             dr-harbor.docmagic.com/docsys/eeligibility-service-cli:master-251852f-042721181924
             harbor.docmagic.com/docsys/eeligibility-service-cli:master-251852f-042721181924
     reporting-service
         torrance
              harbor.docmagic.com/docsys/dsiconfig-init:jpmc-fm1-020822181951
         austin
              dr-harbor.docmagic.com/docsys/dsiconfig-init:fm1-051021191732
         torrance
              harbor.docmagic.com/docsys/dsitemplates-init:jpmc-fm1-022422155417
         austin
              dr-harbor.docmagic.com/docsys/dsitemplates-init:jpmc-fm1-020822182115

wf-prod-torrance  vs  wf-prod-austin
    missing in torrance:  accessibility-api
    missing in torrance:  autoprepeditor-ui
    missing in torrance:  billingevent-service
    missing in torrance:  docmagiclite-ui
    missing in austin  :  evaultstorage-service

     esignpackage-service
         torrance
              harbor.docmagic.com/docsys/esignpackage-service:1.7.5-57b7c00-111521160457
         austin
              dr-harbor.docmagic.com/docsys/esignpackage-service:1.8.1-b6c674d-120121150952
     nmlsupdate-cli
         torrance
             harbor.docmagic.com/docsys/nmlsupdate-cli:1.0-d22848d-100521134007
         austin

Of course @JonSG, you had the right idea, better to do like you did, either not in one or not in the other, instead of missing in one, and then pointing out some differences in both.

2

Answers


  1. Chosen as BEST ANSWER

    The solution to my original post is:

    #!/usr/bin/python3
    import json,urllib.request,difflib,datetime,smtplib,os,email.message,csv,sys,base64
    from email.mime.multipart import MIMEMultipart
    from email.mime.text import MIMEText
    from datetime import date
    from email.message import EmailMessage
    
    diff = difflib.Differ()
    
    def createEnv(name, url):
        env = dict(name=name, url=url)
        return env
    
    def create(torranceEnvName, torranceEnvUrl, austinName, austinUrl):
        torranceEnv = createEnv(torranceEnvName, torranceEnvUrl)
        austinEnv = createEnv(austinName, austinUrl)
        env = dict(torranceEnv=torranceEnv, austinEnv=austinEnv)
        return env
    
    
    
    def normalizeAustinValues(imageTag):
        austinHost = 'dr-harbor.docmagic.com'
        torranceHost = 'harbor.docmagic.com'
        if (austinHost in imageTag):
            imageTag = imageTag.replace(austinHost, torranceHost)
        return imageTag
    
    def compareEnvironmentTags(torranceEnv, austinEnv, report_file):
    
        data_1 = urllib.request.urlopen(torranceEnv['url']).read()
        data_2 = urllib.request.urlopen(austinEnv['url']).read()
    
        torrance_dict = json.loads(data_1)
        austin_dict = json.loads(data_2)
    
        missing_in_torrance = []
        missing_in_austin = []
        check_for_diffs = []
    
        for i in austin_dict:
            if (i not in torrance_dict):
                missing_in_torrance.append(i)
    
        for i in torrance_dict:
            if (i not in austin_dict):
                missing_in_austin.append(i)
            else:
                check_for_diffs.append(i)
    
        for i in missing_in_torrance:
            print('tmissing in torrance: ', i, file=report_file)
        for i in missing_in_austin:
            print('tmissing in austin  : ', i, file=report_file)
    
        print(file=report_file)
        for i in check_for_diffs:
            
            torrance_images = torrance_dict[i][torranceEnv['name']]['images']
            austin_images = austin_dict[i][austinEnv['name']]['images']
    
            torrance_images = list(torrance_images)
            austin_images = list(austin_images)
            normalized_austin_images = list(map(normalizeAustinValues, austin_images))
            
            torrance_images.sort()
            austin_images.sort()
            normalized_austin_images.sort()
    
            result = diff.compare(torrance_images, normalized_austin_images)
            result = [line for line in result if line.startswith(("- ", "+ "))]
            result = ''.join(result)
    
            if (result): 
                print('t', i, file=report_file)
                
                if (len(torrance_images) != len(austin_images)):
                    torrance_not_in_austin = set(torrance_images).difference(normalized_austin_images)
                    austin_not_in_torrance = set(normalized_austin_images).difference(torrance_images)  
                    print('tt torrance', file=report_file)
                    for image in torrance_not_in_austin:
                        print('ttt', image, file=report_file)
                    print('tt austin', file=report_file)
                    for image in austin_not_in_torrance:
                        print('ttt','dr-', image, file=report_file)
                else:
                    for idx,image in enumerate(torrance_images):
                        if (image != normalized_austin_images[idx]):
                            print('tt torrance', file=report_file)
                            print('ttt ', image, file=report_file)
                            print('tt austin', file=report_file)
                            print('ttt ', austin_images[idx], file=report_file)
    
        return #compareEnvironmenTags
    
    ### execution block
    
    environments = []
    environments.append(create("docmagic-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/docmagic-prod-torrance", "docmagic-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/docmagic-prod-austin"))
    environments.append(create("docmagic-dev-torrance", "https://gitops-service.docmagic.com/argocd/compare/docmagic-dev-torrance", "docmagic-dev-austin", "https://gitops-service.docmagic.com/argocd/compare/docmagic-dev-austin"))
    environments.append(create("uwm-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/uwm-prod-torrance", "uwm-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/uwm-prod-austin"))
    environments.append(create("eps-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/eps-prod-torrance", "eps-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/eps-prod-austin"))
    environments.append(create("jpmc-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/jpmc-prod-torrance", "jpmc-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/jpmc-prod-austin"))
    environments.append(create("wf-prod-torrance", "https://gitops-service.docmagic.com/argocd/compare/wf-prod-torrance", "wf-prod-austin", "https://gitops-service.docmagic.com/argocd/compare/wf-prod-austin"))
    
    with open('report.txt', 'w') as report_file:
        for env in environments:
            print(env['torranceEnv']['name'], ' vs ', env['austinEnv']['name'], file=report_file)
            compareEnvironmentTags(env['torranceEnv'], env['austinEnv'], report_file)
            print(file=report_file)
    
    # Sender and recipient
    me = '[email protected]'
    you = '[email protected]'      
    # Open the plain text file whose name is in textfile for reading.
    textfile = 'report.txt'
    with open(textfile) as fp:
        # Create a text/plain message
        msg = EmailMessage()
        msg.set_content(fp.read())
    
    # me == the sender's email address
    # you == the recipient's email address
    date = datetime.datetime.now().strftime('%m-%d-%Y')
    msg['Subject'] = f'Image Diff Report %s' % date
    msg['From'] = me
    msg['To'] = you
    
    # Send the message via our own SMTP server.
    s = smtplib.SMTP('smtp-gateway.docmagic.com:25')
    s.send_message(msg)
    s.quit()
    os.remove("report.txt")
    

  2. I think you might want to leverage set().difference() to get the image in one list that are not in the other and vice versa:

    for torrance_service_key, torrance_service_value in torrance_dict.items():
        torrance_service_images = list(torrance_service_value.values())[0]["images"]
    
        austin_service_value = austin_dict.get(torrance_service_key, {})
        austin_service_images = list(austin_service_value.values())[0]["images"] if list(austin_service_value.values()) else []
        austin_service_images = [i.replace("dr-harbor.docmagic.com", "harbor.docmagic.com") for i in austin_service_images]
    
        torrance_not_in_austin = set(torrance_service_images).difference(austin_service_images)
        austin_not_in_torrance = set(austin_service_images).difference(torrance_service_images)
    
        if torrance_not_in_austin or austin_not_in_torrance:
            print(f"{torrance_service_key}:")
            print("tonly in torrance:")
            for image in torrance_not_in_austin:
                print(f"tt{image}")
    
            print("tonly in austin:")
            for image in austin_not_in_torrance:
                print(f"tt{image}")
    

    Should give you something like:

    cascustomer-service:
            only in torrance:
            only in austin:
                    harbor.docmagic.com/library/dm-nginx:latest
                    harbor.docmagic.com/docsys/cascustomer-service:4.8.0-SNAPSHOT-20210623172337-062321102337
    
    Login or Signup to reply.
Please signup or login to give your own answer.
Back To Top
Search