# -*- coding: utf-8 -*- import boto3 import time _dict = { # 6910账号aksk 'AWS_ACCESS_KEY_ID' : 'AKIA4FU4JFDXO5J2ISNY', 'AWS_SECRET_ACCESS_KEY': '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', # 7422账号aksk 'GLOBAL_AWS_ACCESS_KEY_ID' : "AKIAQWIJ3GB7CTO5GAWY", 'GLOBAL_AWS_SECRET_ACCESS_KEY' : "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", } aws_aksk = { "933673652629": { "account_type": "amazon-standard", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "aws-shaklee-billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws:iam::933673652629:role/GlobalMSP-Shaklee-BillingPlatform", "external_id": "l2c-4s8sa0lu" }, "314136173648": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "shakleebilling", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::314136173648:role/l2c-aaege4ot", "external_id": "l2c-aaege4ot" }, # shake # "190841870523": { # "account_type": "amazon-china", # "aws_access_key_id": "AKIASY3YP3C54BIQHAGH", # "aws_secret_access_key": "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", # "bucket": "billing-cost-allocate", # "credential_type": "Amazon Access Key", # "role_arn": "", # "external_id": "" # }, # "190841870523": { # "account_type": "amazon-china", # "aws_access_key_id": "AKIASY3YP3C54BIQHAGH", # "aws_secret_access_key": "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", # "bucket": "billing-cost-allocate", # "credential_type": "Amazon Access Key", # "role_arn": "", # "external_id": "" # }, "190508708273": { "account_type": "amazon-china", "aws_access_key_id": "AKIASYWZUGGY6HXX7L7X", "aws_secret_access_key": "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", "bucket": "billing-bucket-190508708273", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "785781180852": { "account_type": "amazon-china", "aws_access_key_id": "AKIA3N5BVJG2B7BN7D7F", "aws_secret_access_key": "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", "bucket": "785781180852billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "936739285320": { "account_type": "amazon-china", "aws_access_key_id": "AKIA5UGQDFFEKWW3I4RG", "aws_secret_access_key": "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", "bucket": "billing-location", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "441271182313": { "account_type": "amazon-china", "aws_access_key_id": "AKIAWNPOP4PURVNFVZBL", "aws_secret_access_key": "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", "bucket": "fc-billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "785691964566": { "account_type": "amazon-china", "aws_access_key_id": "AKIA3N3XD6SLNPLCOS3H", "aws_secret_access_key": "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", "bucket": "785691964566billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "086031976377": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "086031976377billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::086031976377:role/086031976377billing", "external_id": "l2c-2zz7ggpv", }, "089158463905": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "089158463905billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::089158463905:role/089158463905billing", "external_id": "l2c-1i5u1ds0", }, "563347375094": { "account_type": "amazon-china", "aws_access_key_id": "AKIAYGKQ6YP3KYLFHVPR", "aws_secret_access_key": "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", "bucket": "563347375094billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "", }, "563646727715": { "account_type": "amazon-china", "aws_access_key_id": "AKIAYGO7WQYRZB2PPXGV", "aws_secret_access_key": "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", "bucket": "563646727715billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "", }, "565095721352": { "account_type": "amazon-china", "aws_access_key_id": "AKIAYHESUNGEDHLQBUXJ", "aws_secret_access_key": "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", "bucket": "565095721352billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "", }, "565359735310": { "account_type": "amazon-china", "aws_access_key_id": "AKIAYHIQQ6IHI4FEZ6TC", "aws_secret_access_key": "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", "bucket": "565359735310billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "", }, "047802167422": { "account_type": "amazon-standard", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "hongkong1-billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws:iam::047802167422:role/GlobalPayer-HongKong1-BillingPlatform", "external_id": "l2c-wkr9pgcc" }, "126733225402": { "account_type": "amazon-standard", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "126733225402billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws:iam::126733225402:role/GlobalPayer-HongKong4-BillingPlatform", "external_id": "l2c-s9vfp4bv" }, "854013721154":{ "account_type": "amazon-china", "aws_access_key_id": "AKIA4NVZPHJBG2UWAIVY", "aws_secret_access_key": "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", "bucket": "jsfund-cost", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "", }, # "785691964566": { # "account_type": "amazon-china", # "aws_access_key_id": "AKIA3N3XD6SLNPLCOS3H", # "aws_secret_access_key": "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", # "bucket": "785691964566billing", # "credential_type": "Amazon Access Key", # "role_arn": "", # "external_id": "" # }, "793775385697": { "account_type": "amazon-standard", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "hongkong5-billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws:iam::793775385697:role/GlobalPayer-HongKong5-BillingPlatform", "external_id": "l2c-8rjrz3by" }, "028328046529": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "028328046529billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::028328046529:role/shinaide", "external_id": "l2c-ujbkbfoz", }, "089027636324": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "089027636324billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::089027636324:role/089027636324billing", "external_id": "l2c-1i5u1ds0", }, "087353436373": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "087353436373billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::087353436373:role/087353436373billing", "external_id": "l2c-bijnnyaf" }, "606073735846":{ "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "087353436373billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::087353436373:role/087353436373billing", "external_id": "l2c-bijnnyaf" } } def auto_download_file_form_s3(file: tuple): for i in file: payer_id = i.split('-')[0] credential = get_credentials(payer_id) bucket = credential.pop('bucket') s3_client = boto3.client('s3', **credential) print('{}开始下载'.format(i)) s3_client.download_file(bucket, i, i) print('{}下载完成'.format(i)) def get_credentials(payer_id): credential = aws_aksk.get(payer_id, None) if not credential: return if credential.get('account_type', None) == 'amazon-china': region_name = 'cn-northwest-1' aws_access_key_id = _dict['AWS_ACCESS_KEY_ID'] aws_secret_access_key = _dict['AWS_SECRET_ACCESS_KEY'] else: region_name = 'us-east-2' aws_access_key_id = _dict['GLOBAL_AWS_ACCESS_KEY_ID'] aws_secret_access_key = _dict['GLOBAL_AWS_SECRET_ACCESS_KEY'] if credential.get('credential_type', None) == 'Amazon Access Key': return { 'aws_access_key_id': credential['aws_access_key_id'], 'aws_secret_access_key': credential['aws_secret_access_key'], 'region_name' : region_name, 'bucket': credential['bucket'] } else: sts = boto3.client('sts', region_name=region_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key ) print(credential['role_arn']) sts_credential = sts.assume_role( RoleArn=credential['role_arn'], RoleSessionName='crawl-billing' + str(time.time()), DurationSeconds=900, ExternalId=credential['external_id'] ) return { 'aws_access_key_id': sts_credential['Credentials']['AccessKeyId'], 'aws_secret_access_key': sts_credential['Credentials']['SecretAccessKey'], 'region_name' : region_name, 'aws_session_token': sts_credential['Credentials']['SessionToken'], 'bucket': credential['bucket'] } if __name__ == '__main__': date = '2024-06' filenames = ( #嘉实 '087353436373-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-'+date+'.csv.zip', '087353436373-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-'+date+'.csv.zip', '126733225402-aws-cost-allocation-'+date+'.csv', ) auto_download_file_form_s3(filenames)