# -*- coding: utf-8 -*- import boto3 import time _dict = { # 6910账号aksk 'AWS_ACCESS_KEY_ID': 'AKIA4FU4JFDXO5J2ISNY', 'AWS_SECRET_ACCESS_KEY': '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', # 7422账号aksk 'GLOBAL_AWS_ACCESS_KEY_ID': "AKIAQWIJ3GB7CTO5GAWY", 'GLOBAL_AWS_SECRET_ACCESS_KEY': "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" } aws_aksk = { "933673652629": { "account_type": "amazon-standard", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "aws-shaklee-billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws:iam::933673652629:role/GlobalMSP-Shaklee-BillingPlatform", "external_id": "l2c-4s8sa0lu" }, "435857483079":{ "account_type": "amazon-standard", "aws_access_key_id": "AKIAQWIJ3GB7CTO5GAWY", "aws_secret_access_key": "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", "bucket": "435857483079billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws:iam::435857483079:role/435857483079-billing", "external_id": "l2c-bqoc69xr" }, "314136173648": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "shakleebilling", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::314136173648:role/l2c-aaege4ot", "external_id": "l2c-aaege4ot" }, # shake # "190841870523": { # "account_type": "amazon-china", # "aws_access_key_id": "AKIASY3YP3C54BIQHAGH", # "aws_secret_access_key": "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", # "bucket": "billing-cost-allocate", # "credential_type": "Amazon Access Key", # "role_arn": "", # "external_id": "" # }, # "190841870523": { # "account_type": "amazon-china", # "aws_access_key_id": "AKIASY3YP3C54BIQHAGH", # "aws_secret_access_key": "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", # "bucket": "billing-cost-allocate", # "credential_type": "Amazon Access Key", # "role_arn": "", # "external_id": "" # }, "190508708273": { "account_type": "amazon-china", "aws_access_key_id": "AKIASYWZUGGY6HXX7L7X", "aws_secret_access_key": "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", "bucket": "billing-bucket-190508708273", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, # "785781180852": { # "account_type": "amazon-china", # "aws_access_key_id": "AKIA3N5BVJG2B7BN7D7F", # "aws_secret_access_key": "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", # "bucket": "785781180852billing", # "credential_type": "Amazon Access Key", # "role_arn": "", # "external_id": "" # }, "936739285320": { "account_type": "amazon-china", "aws_access_key_id": "AKIA5UGQDFFEKWW3I4RG", "aws_secret_access_key": "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", "bucket": "billing-location", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "441271182313": { "account_type": "amazon-china", "aws_access_key_id": "AKIAWNPOP4PURVNFVZBL", "aws_secret_access_key": "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", "bucket": "fc-billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "785691964566": { "account_type": "amazon-china", "aws_access_key_id": "AKIA3N3XD6SLNPLCOS3H", "aws_secret_access_key": "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", "bucket": "785691964566billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "086031976377": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "086031976377billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::086031976377:role/086031976377billing", "external_id": "l2c-2zz7ggpv", }, "089158463905": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "089158463905billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::089158463905:role/089158463905billing", "external_id": "l2c-1i5u1ds0", }, "563347375094": { "account_type": "amazon-china", "aws_access_key_id": "AKIAYGKQ6YP3KYLFHVPR", "aws_secret_access_key": "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", "bucket": "563347375094billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "", }, "563646727715": { "account_type": "amazon-china", "aws_access_key_id": "AKIAYGO7WQYRZB2PPXGV", "aws_secret_access_key": "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", "bucket": "563646727715billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "", }, "565095721352": { "account_type": "amazon-china", "aws_access_key_id": "AKIAYHESUNGEDHLQBUXJ", "aws_secret_access_key": "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", "bucket": "565095721352billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "", }, "565359735310": { "account_type": "amazon-china", "aws_access_key_id": "AKIAYHIQQ6IHI4FEZ6TC", "aws_secret_access_key": "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", "bucket": "565359735310billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "", }, "047802167422": { "account_type": "amazon-standard", "aws_access_key_id": "AKIAQWIJ3GB7CTO5GAWY", "aws_secret_access_key": "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", "bucket": "hongkong1-billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws:iam::047802167422:role/GlobalPayer-HongKong1-BillingPlatform", "external_id": "l2c-wkr9pgcc" }, "126733225402": { "account_type": "amazon-standard", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "126733225402billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws:iam::126733225402:role/GlobalPayer-HongKong4-BillingPlatform", "external_id": "l2c-s9vfp4bv" }, # "785691964566": { # "account_type": "amazon-china", # "aws_access_key_id": "AKIA3N3XD6SLNPLCOS3H", # "aws_secret_access_key": "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", # "bucket": "785691964566billing", # "credential_type": "Amazon Access Key", # "role_arn": "", # "external_id": "" # }, "793775385697": { "account_type": "amazon-standard", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "hongkong5-billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws:iam::793775385697:role/GlobalPayer-HongKong5-BillingPlatform", "external_id": "l2c-8rjrz3by" }, "028328046529": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "aws-billing-test02", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::028328046529:role/shinaide", "external_id": "l2c-ujbkbfoz", }, "010265887827": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "010265887827billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::010265887827:role/daimler", "external_id": "l2c-4pv6ood4" }, "011562250191": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "011562250191billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::011562250191:role/daimler", "external_id": "l2c-4pv6ood4" }, "011276597621": { "account_type": "amazon-china", "aws_access_key_id": "", "aws_secret_access_key": "", "bucket": "011276597621billing", "credential_type": "Amazon IAM Role", "role_arn": "arn:aws-cn:iam::011276597621:role/daimler", "external_id": "l2c-4pv6ood4" }, "782891169312": { "account_type": "amazon-china", "aws_access_key_id": "AKIA3MR7TGYQDORRLVOS", "aws_secret_access_key": "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", "bucket": "782891169312billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "606073735846":{ "account_type": "amazon-china", "aws_access_key_id": "AKIA4FU4JFDXO5J2ISNY", "aws_secret_access_key": "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", "bucket": "087353436373billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" }, "690337281734":{ "account_type": "amazon-china", "aws_access_key_id": "AKIAQWIJ3GB7CTO5GAWY", "aws_secret_access_key": "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", "bucket": "hongkong1-billing", "credential_type": "Amazon Access Key", "role_arn": "", "external_id": "" } } def getFiles(payer_id): credential = get_credentials(payer_id) bucket = credential.pop('bucket') s3 = boto3.resource('s3' , **credential) bucket = s3.Bucket('087353436373billing') objetss=bucket.objects.limit(10000); for key in objetss: print(key.key) def getBucket(payer_id): credential = get_credentials(payer_id) bucket = credential.pop('bucket') s3 = boto3.resource('s3' , **credential) bucket = list(s3.buckets.all()) for key in bucket: print(key) def auto_download_file_form_s3(file: tuple): for i in file: payer_id = i.split('-')[0] credential = get_credentials(payer_id) bucket = credential.pop('bucket') s3_client = boto3.client('s3', **credential) print('{}开始下载'.format(i)) # s3_client.download_file s3_client.download_file( bucket, i, i) print('{}下载完成'.format(i)) def get_credentials(payer_id): credential = aws_aksk.get(payer_id, None) if not credential: return if credential.get('account_type', None) == 'amazon-china': region_name = 'cn-northwest-1' aws_access_key_id = _dict['AWS_ACCESS_KEY_ID'] aws_secret_access_key = _dict['AWS_SECRET_ACCESS_KEY'] else: region_name = 'us-east-2' aws_access_key_id = _dict['GLOBAL_AWS_ACCESS_KEY_ID'] aws_secret_access_key = _dict['GLOBAL_AWS_SECRET_ACCESS_KEY'] if credential.get('credential_type', None) == 'Amazon Access Key': return { 'aws_access_key_id': credential['aws_access_key_id'], 'aws_secret_access_key': credential['aws_secret_access_key'], 'region_name': region_name, 'bucket': credential['bucket'] } else: sts = boto3.client('sts', region_name=region_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key ) sts_credential = sts.assume_role( RoleArn=credential['role_arn'], RoleSessionName='crawl-billing' + str(time.time()), DurationSeconds=900, ExternalId=credential['external_id'] ) return { 'aws_access_key_id': sts_credential['Credentials']['AccessKeyId'], 'aws_secret_access_key': sts_credential['Credentials']['SecretAccessKey'], 'region_name': region_name, 'aws_session_token': sts_credential['Credentials']['SessionToken'], 'bucket': credential['bucket'] } import sys if __name__ == '__main__': # aws - billing - test - daily / aws - billing - test01 / 20230901 - 20231001 / aws - billing - test01 - 00001. # csv.gz # aws - billing - test - daily / aws - billing - test01 / 20230901 - 20231001 / aws - billing - test01 - 00002. # csv.gz filenames = ( # daimler # '563347375094-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip', # '563347375094-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip', # '028328046529-aws-billing-test-daily/aws-billing-test01/20230901-20231001/028328046529-aws-billing-test01-00001.csv.gz', # '028328046529-aws-billing-test01-00002.csv.gz', # '028328046529-aws-billing-detailed-line-items-ACTS-2023-09.csv.zip', # '028328046529-aws-billing-detailed-line-items-ACTS-Ningxia-2023-09.csv.zip', # '563646727715-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip', # '563646727715-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip', # '565095721352-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip', # '565095721352-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip', # '565359735310-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip', # '565359735310-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip', # '010265887827-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip', # '010265887827-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip', # '011562250191-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip', # '011562250191-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip', # '011276597621-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip', # '011276597621-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip' # '047802167422-aws-cost-allocation-ACTS-2023-09.csv', # '047802167422-aws-cost-allocation-ACTS-Ningxia-2023-09.csv', # '047802167422-aws-cost-allocation-2023-09.csv' '087353436373-aws-cost-allocation-ACTS-2024-06.csv' '087353436373-aws-cost-allocation-ACTS-Ningxia-2024-03.csv', '087353436373-aws-cost-allocation-2024-03.csv', '087353436373-aws-cost-allocation-AWSMarketplace-2024-03.csv', # 施耐德 # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2022-08.csv.zip', # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-04.csv.zip', # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2022-07.csv.zip', # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2022-07.csv.zip', # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2022-02.csv.zip', # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2022-02.csv.zip', # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2022-03.csv.zip', # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2022-03.csv.zip', # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip', # '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip', ) # getBucket('435857483079') # getFiles('435857483079') # auto_download_file_form_s3(filenames) # # getBucket('028328046529') getFiles('087353436373')