You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
295 lines
11 KiB
295 lines
11 KiB
# -*- coding: utf-8 -*-
|
|
import boto3
|
|
import time
|
|
|
|
_dict = {
|
|
# 6910账号aksk
|
|
'AWS_ACCESS_KEY_ID' : 'AKIA4FU4JFDXO5J2ISNY',
|
|
'AWS_SECRET_ACCESS_KEY': 'AVfPZ/XryDsoWmYB293eNPOyiO82x9xR62KFKezS',
|
|
# 7422账号aksk
|
|
'GLOBAL_AWS_ACCESS_KEY_ID' : "AKIAQWIJ3GB7CTO5GAWY",
|
|
'GLOBAL_AWS_SECRET_ACCESS_KEY' : "o1iQFQFSW+W/GGzjIRcNu1RDO1s72Uv2pJK/zf3z",
|
|
}
|
|
|
|
aws_aksk = {
|
|
"933673652629": {
|
|
"account_type": "amazon-standard",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "aws-shaklee-billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws:iam::933673652629:role/GlobalMSP-Shaklee-BillingPlatform",
|
|
"external_id": "l2c-4s8sa0lu"
|
|
},
|
|
"314136173648": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "shakleebilling",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::314136173648:role/l2c-aaege4ot",
|
|
"external_id": "l2c-aaege4ot"
|
|
},
|
|
# shake
|
|
# "190841870523": {
|
|
# "account_type": "amazon-china",
|
|
# "aws_access_key_id": "AKIASY3YP3C54BIQHAGH",
|
|
# "aws_secret_access_key": "FbGi15ZDnmM9/i17889RzzcclS5uGgNzi01btMxu",
|
|
# "bucket": "billing-cost-allocate",
|
|
# "credential_type": "Amazon Access Key",
|
|
# "role_arn": "",
|
|
# "external_id": ""
|
|
# },
|
|
|
|
# "190841870523": {
|
|
# "account_type": "amazon-china",
|
|
# "aws_access_key_id": "AKIASY3YP3C54BIQHAGH",
|
|
# "aws_secret_access_key": "FbGi15ZDnmM9/i17889RzzcclS5uGgNzi01btMxu",
|
|
# "bucket": "billing-cost-allocate",
|
|
# "credential_type": "Amazon Access Key",
|
|
# "role_arn": "",
|
|
# "external_id": ""
|
|
# },
|
|
"190508708273": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIASYWZUGGY6HXX7L7X",
|
|
"aws_secret_access_key": "W451fxDRdBsxHH9taXO+KtTmy4YckOw98IngMNh4",
|
|
"bucket": "billing-bucket-190508708273",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"785781180852": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIA3N5BVJG2B7BN7D7F",
|
|
"aws_secret_access_key": "FgNR1LMEfW2Z3imL4PTZCGyslxjz4HV5m4Ut4hm5",
|
|
"bucket": "785781180852billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"936739285320": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIA5UGQDFFEKWW3I4RG",
|
|
"aws_secret_access_key": "wpBOnLxZSfgrazkip+0f2Lh2lX/PzHmgK/jPni5V",
|
|
"bucket": "billing-location",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"441271182313": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAWNPOP4PURVNFVZBL",
|
|
"aws_secret_access_key": "vhic5R4U586pe01Ib3e0zzVkZgi8Qj3Cqa3h7abe",
|
|
"bucket": "fc-billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"785691964566": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIA3N3XD6SLNPLCOS3H",
|
|
"aws_secret_access_key": "4x1/mQawUu6NR93uD1wppY+fuBh+XomxVRB0wUu/",
|
|
"bucket": "785691964566billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"086031976377": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "086031976377billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::086031976377:role/086031976377billing",
|
|
"external_id": "l2c-2zz7ggpv",
|
|
},
|
|
"089158463905": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "089158463905billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::089158463905:role/089158463905billing",
|
|
"external_id": "l2c-1i5u1ds0",
|
|
},
|
|
"563347375094": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAYGKQ6YP3KYLFHVPR",
|
|
"aws_secret_access_key": "q5gnDXw03SxgDh3xc5YyQ8Z80tQtcr3FIcHmxANh",
|
|
"bucket": "563347375094billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": "",
|
|
},
|
|
"563646727715": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAYGO7WQYRZB2PPXGV",
|
|
"aws_secret_access_key": "mWFlRdM4t81YBwege9us3N5GRwRUbmCDmcDx9U24",
|
|
"bucket": "563646727715billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": "",
|
|
},
|
|
"565095721352": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAYHESUNGEDHLQBUXJ",
|
|
"aws_secret_access_key": "x+FP8vjr/6NoQAkzpnvqs6Qe3gWr3TFLZXRcU0e0",
|
|
"bucket": "565095721352billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": "",
|
|
},
|
|
"565359735310": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAYHIQQ6IHI4FEZ6TC",
|
|
"aws_secret_access_key": "JdfjSpbwoZwdhb3tXi2QYUrzfUH+01BrWCM6IB32",
|
|
"bucket": "565359735310billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": "",
|
|
},
|
|
"047802167422": {
|
|
"account_type": "amazon-standard",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "hongkong1-billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws:iam::047802167422:role/GlobalPayer-HongKong1-BillingPlatform",
|
|
"external_id": "l2c-wkr9pgcc"
|
|
},
|
|
"126733225402": {
|
|
"account_type": "amazon-standard",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "126733225402billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws:iam::126733225402:role/GlobalPayer-HongKong4-BillingPlatform",
|
|
"external_id": "l2c-s9vfp4bv"
|
|
},
|
|
"854013721154":{
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIA4NVZPHJBG2UWAIVY",
|
|
"aws_secret_access_key": "dqL3BimlCU/IUdZuSz14Cqrk5CHzOMOYlyA0k/o4",
|
|
"bucket": "jsfund-cost",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": "",
|
|
},
|
|
# "785691964566": {
|
|
# "account_type": "amazon-china",
|
|
# "aws_access_key_id": "AKIA3N3XD6SLNPLCOS3H",
|
|
# "aws_secret_access_key": "4x1/mQawUu6NR93uD1wppY+fuBh+XomxVRB0wUu/",
|
|
# "bucket": "785691964566billing",
|
|
# "credential_type": "Amazon Access Key",
|
|
# "role_arn": "",
|
|
# "external_id": ""
|
|
# },
|
|
"793775385697": {
|
|
"account_type": "amazon-standard",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "hongkong5-billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws:iam::793775385697:role/GlobalPayer-HongKong5-BillingPlatform",
|
|
"external_id": "l2c-8rjrz3by"
|
|
},
|
|
"028328046529": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "028328046529billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::028328046529:role/shinaide",
|
|
"external_id": "l2c-ujbkbfoz",
|
|
},
|
|
"089027636324": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "089027636324billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::089027636324:role/089027636324billing",
|
|
"external_id": "l2c-1i5u1ds0",
|
|
},
|
|
"087353436373": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "087353436373billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::087353436373:role/087353436373billing",
|
|
"external_id": "l2c-bijnnyaf"
|
|
},
|
|
"606073735846":{
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "087353436373billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::087353436373:role/087353436373billing",
|
|
"external_id": "l2c-bijnnyaf"
|
|
}
|
|
}
|
|
|
|
def auto_download_file_form_s3(file: tuple):
|
|
for i in file:
|
|
payer_id = i.split('-')[0]
|
|
credential = get_credentials(payer_id)
|
|
bucket = credential.pop('bucket')
|
|
s3_client = boto3.client('s3', **credential)
|
|
print('{}开始下载'.format(i))
|
|
s3_client.download_file(bucket, i, i)
|
|
print('{}下载完成'.format(i))
|
|
|
|
|
|
def get_credentials(payer_id):
|
|
credential = aws_aksk.get(payer_id, None)
|
|
if not credential:
|
|
return
|
|
if credential.get('account_type', None) == 'amazon-china':
|
|
region_name = 'cn-northwest-1'
|
|
aws_access_key_id = _dict['AWS_ACCESS_KEY_ID']
|
|
aws_secret_access_key = _dict['AWS_SECRET_ACCESS_KEY']
|
|
else:
|
|
region_name = 'us-east-2'
|
|
aws_access_key_id = _dict['GLOBAL_AWS_ACCESS_KEY_ID']
|
|
aws_secret_access_key = _dict['GLOBAL_AWS_SECRET_ACCESS_KEY']
|
|
if credential.get('credential_type', None) == 'Amazon Access Key':
|
|
return {
|
|
'aws_access_key_id': credential['aws_access_key_id'],
|
|
'aws_secret_access_key': credential['aws_secret_access_key'],
|
|
'region_name' : region_name,
|
|
'bucket': credential['bucket']
|
|
}
|
|
else:
|
|
sts = boto3.client('sts',
|
|
region_name=region_name,
|
|
aws_access_key_id=aws_access_key_id,
|
|
aws_secret_access_key=aws_secret_access_key
|
|
)
|
|
print(credential['role_arn'])
|
|
sts_credential = sts.assume_role(
|
|
RoleArn=credential['role_arn'],
|
|
RoleSessionName='crawl-billing' + str(time.time()),
|
|
DurationSeconds=900,
|
|
ExternalId=credential['external_id']
|
|
)
|
|
return {
|
|
'aws_access_key_id': sts_credential['Credentials']['AccessKeyId'],
|
|
'aws_secret_access_key': sts_credential['Credentials']['SecretAccessKey'],
|
|
'region_name' : region_name,
|
|
'aws_session_token': sts_credential['Credentials']['SessionToken'],
|
|
'bucket': credential['bucket']
|
|
}
|
|
|
|
if __name__ == '__main__':
|
|
|
|
date = '2024-06'
|
|
filenames = (
|
|
#嘉实
|
|
'087353436373-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-'+date+'.csv.zip',
|
|
'087353436373-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-'+date+'.csv.zip',
|
|
'126733225402-aws-cost-allocation-'+date+'.csv',
|
|
)
|
|
auto_download_file_form_s3(filenames) |