You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
386 lines
16 KiB
386 lines
16 KiB
# -*- coding: utf-8 -*-
|
|
import boto3
|
|
import time
|
|
|
|
_dict = {
|
|
# 6910账号aksk
|
|
'AWS_ACCESS_KEY_ID': 'AKIA4FU4JFDXO5J2ISNY',
|
|
'AWS_SECRET_ACCESS_KEY': 'AVfPZ/XryDsoWmYB293eNPOyiO82x9xR62KFKezS',
|
|
# 7422账号aksk
|
|
'GLOBAL_AWS_ACCESS_KEY_ID': "AKIAQWIJ3GB7CTO5GAWY",
|
|
'GLOBAL_AWS_SECRET_ACCESS_KEY': "o1iQFQFSW+W/GGzjIRcNu1RDO1s72Uv2pJK/zf3z"
|
|
}
|
|
|
|
aws_aksk = {
|
|
|
|
"933673652629": {
|
|
"account_type": "amazon-standard",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "aws-shaklee-billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws:iam::933673652629:role/GlobalMSP-Shaklee-BillingPlatform",
|
|
"external_id": "l2c-4s8sa0lu"
|
|
},
|
|
"435857483079":{
|
|
"account_type": "amazon-standard",
|
|
"aws_access_key_id": "AKIAQWIJ3GB7CTO5GAWY",
|
|
"aws_secret_access_key": "o1iQFQFSW+W/GGzjIRcNu1RDO1s72Uv2pJK/zf3z",
|
|
"bucket": "435857483079billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws:iam::435857483079:role/435857483079-billing",
|
|
"external_id": "l2c-bqoc69xr"
|
|
},
|
|
"314136173648": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "shakleebilling",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::314136173648:role/l2c-aaege4ot",
|
|
"external_id": "l2c-aaege4ot"
|
|
},
|
|
# shake
|
|
# "190841870523": {
|
|
# "account_type": "amazon-china",
|
|
# "aws_access_key_id": "AKIASY3YP3C54BIQHAGH",
|
|
# "aws_secret_access_key": "FbGi15ZDnmM9/i17889RzzcclS5uGgNzi01btMxu",
|
|
# "bucket": "billing-cost-allocate",
|
|
# "credential_type": "Amazon Access Key",
|
|
# "role_arn": "",
|
|
# "external_id": ""
|
|
# },
|
|
|
|
# "190841870523": {
|
|
# "account_type": "amazon-china",
|
|
# "aws_access_key_id": "AKIASY3YP3C54BIQHAGH",
|
|
# "aws_secret_access_key": "FbGi15ZDnmM9/i17889RzzcclS5uGgNzi01btMxu",
|
|
# "bucket": "billing-cost-allocate",
|
|
# "credential_type": "Amazon Access Key",
|
|
# "role_arn": "",
|
|
# "external_id": ""
|
|
# },
|
|
"190508708273": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIASYWZUGGY6HXX7L7X",
|
|
"aws_secret_access_key": "W451fxDRdBsxHH9taXO+KtTmy4YckOw98IngMNh4",
|
|
"bucket": "billing-bucket-190508708273",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
# "785781180852": {
|
|
# "account_type": "amazon-china",
|
|
# "aws_access_key_id": "AKIA3N5BVJG2B7BN7D7F",
|
|
# "aws_secret_access_key": "FgNR1LMEfW2Z3imL4PTZCGyslxjz4HV5m4Ut4hm5",
|
|
# "bucket": "785781180852billing",
|
|
# "credential_type": "Amazon Access Key",
|
|
# "role_arn": "",
|
|
# "external_id": ""
|
|
# },
|
|
"936739285320": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIA5UGQDFFEKWW3I4RG",
|
|
"aws_secret_access_key": "wpBOnLxZSfgrazkip+0f2Lh2lX/PzHmgK/jPni5V",
|
|
"bucket": "billing-location",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"441271182313": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAWNPOP4PURVNFVZBL",
|
|
"aws_secret_access_key": "vhic5R4U586pe01Ib3e0zzVkZgi8Qj3Cqa3h7abe",
|
|
"bucket": "fc-billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"785691964566": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIA3N3XD6SLNPLCOS3H",
|
|
"aws_secret_access_key": "4x1/mQawUu6NR93uD1wppY+fuBh+XomxVRB0wUu/",
|
|
"bucket": "785691964566billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"086031976377": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "086031976377billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::086031976377:role/086031976377billing",
|
|
"external_id": "l2c-2zz7ggpv",
|
|
},
|
|
"089158463905": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "089158463905billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::089158463905:role/089158463905billing",
|
|
"external_id": "l2c-1i5u1ds0",
|
|
},
|
|
"563347375094": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAYGKQ6YP3KYLFHVPR",
|
|
"aws_secret_access_key": "q5gnDXw03SxgDh3xc5YyQ8Z80tQtcr3FIcHmxANh",
|
|
"bucket": "563347375094billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": "",
|
|
},
|
|
"563646727715": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAYGO7WQYRZB2PPXGV",
|
|
"aws_secret_access_key": "mWFlRdM4t81YBwege9us3N5GRwRUbmCDmcDx9U24",
|
|
"bucket": "563646727715billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": "",
|
|
},
|
|
"565095721352": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAYHESUNGEDHLQBUXJ",
|
|
"aws_secret_access_key": "x+FP8vjr/6NoQAkzpnvqs6Qe3gWr3TFLZXRcU0e0",
|
|
"bucket": "565095721352billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": "",
|
|
},
|
|
"565359735310": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAYHIQQ6IHI4FEZ6TC",
|
|
"aws_secret_access_key": "JdfjSpbwoZwdhb3tXi2QYUrzfUH+01BrWCM6IB32",
|
|
"bucket": "565359735310billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": "",
|
|
},
|
|
"047802167422": {
|
|
"account_type": "amazon-standard",
|
|
"aws_access_key_id": "AKIAQWIJ3GB7CTO5GAWY",
|
|
"aws_secret_access_key": "o1iQFQFSW+W/GGzjIRcNu1RDO1s72Uv2pJK/zf3z",
|
|
"bucket": "hongkong1-billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws:iam::047802167422:role/GlobalPayer-HongKong1-BillingPlatform",
|
|
"external_id": "l2c-wkr9pgcc"
|
|
},
|
|
"126733225402": {
|
|
"account_type": "amazon-standard",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "126733225402billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws:iam::126733225402:role/GlobalPayer-HongKong4-BillingPlatform",
|
|
"external_id": "l2c-s9vfp4bv"
|
|
},
|
|
# "785691964566": {
|
|
# "account_type": "amazon-china",
|
|
# "aws_access_key_id": "AKIA3N3XD6SLNPLCOS3H",
|
|
# "aws_secret_access_key": "4x1/mQawUu6NR93uD1wppY+fuBh+XomxVRB0wUu/",
|
|
# "bucket": "785691964566billing",
|
|
# "credential_type": "Amazon Access Key",
|
|
# "role_arn": "",
|
|
# "external_id": ""
|
|
# },
|
|
"793775385697": {
|
|
"account_type": "amazon-standard",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "hongkong5-billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws:iam::793775385697:role/GlobalPayer-HongKong5-BillingPlatform",
|
|
"external_id": "l2c-8rjrz3by"
|
|
},
|
|
"028328046529": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "aws-billing-test02",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::028328046529:role/shinaide",
|
|
"external_id": "l2c-ujbkbfoz",
|
|
},
|
|
"010265887827": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "010265887827billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::010265887827:role/daimler",
|
|
"external_id": "l2c-4pv6ood4"
|
|
},
|
|
"011562250191": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "011562250191billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::011562250191:role/daimler",
|
|
"external_id": "l2c-4pv6ood4"
|
|
},
|
|
"011276597621": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "",
|
|
"aws_secret_access_key": "",
|
|
"bucket": "011276597621billing",
|
|
"credential_type": "Amazon IAM Role",
|
|
"role_arn": "arn:aws-cn:iam::011276597621:role/daimler",
|
|
"external_id": "l2c-4pv6ood4"
|
|
},
|
|
"782891169312": {
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIA3MR7TGYQDORRLVOS",
|
|
"aws_secret_access_key": "JY5bUcDtkqcOWRhBSmNQCBJopZlsgNZZJRnKrPuf",
|
|
"bucket": "782891169312billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"606073735846":{
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIA4FU4JFDXO5J2ISNY",
|
|
"aws_secret_access_key": "AVfPZ/XryDsoWmYB293eNPOyiO82x9xR62KFKezS",
|
|
"bucket": "087353436373billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
},
|
|
"690337281734":{
|
|
"account_type": "amazon-china",
|
|
"aws_access_key_id": "AKIAQWIJ3GB7CTO5GAWY",
|
|
"aws_secret_access_key": "o1iQFQFSW+W/GGzjIRcNu1RDO1s72Uv2pJK/zf3z",
|
|
"bucket": "hongkong1-billing",
|
|
"credential_type": "Amazon Access Key",
|
|
"role_arn": "",
|
|
"external_id": ""
|
|
}
|
|
}
|
|
|
|
def getFiles(payer_id):
|
|
credential = get_credentials(payer_id)
|
|
bucket = credential.pop('bucket')
|
|
s3 = boto3.resource('s3' , **credential)
|
|
bucket = s3.Bucket('087353436373billing')
|
|
objetss=bucket.objects.limit(10000);
|
|
|
|
for key in objetss:
|
|
print(key.key)
|
|
def getBucket(payer_id):
|
|
credential = get_credentials(payer_id)
|
|
bucket = credential.pop('bucket')
|
|
s3 = boto3.resource('s3' , **credential)
|
|
bucket = list(s3.buckets.all())
|
|
for key in bucket:
|
|
print(key)
|
|
|
|
|
|
def auto_download_file_form_s3(file: tuple):
|
|
for i in file:
|
|
payer_id = i.split('-')[0]
|
|
credential = get_credentials(payer_id)
|
|
bucket = credential.pop('bucket')
|
|
s3_client = boto3.client('s3', **credential)
|
|
print('{}开始下载'.format(i))
|
|
# s3_client.download_file
|
|
s3_client.download_file(
|
|
bucket, i, i)
|
|
print('{}下载完成'.format(i))
|
|
|
|
|
|
def get_credentials(payer_id):
|
|
credential = aws_aksk.get(payer_id, None)
|
|
if not credential:
|
|
return
|
|
if credential.get('account_type', None) == 'amazon-china':
|
|
region_name = 'cn-northwest-1'
|
|
aws_access_key_id = _dict['AWS_ACCESS_KEY_ID']
|
|
aws_secret_access_key = _dict['AWS_SECRET_ACCESS_KEY']
|
|
else:
|
|
region_name = 'us-east-2'
|
|
aws_access_key_id = _dict['GLOBAL_AWS_ACCESS_KEY_ID']
|
|
aws_secret_access_key = _dict['GLOBAL_AWS_SECRET_ACCESS_KEY']
|
|
if credential.get('credential_type', None) == 'Amazon Access Key':
|
|
return {
|
|
'aws_access_key_id': credential['aws_access_key_id'],
|
|
'aws_secret_access_key': credential['aws_secret_access_key'],
|
|
'region_name': region_name,
|
|
'bucket': credential['bucket']
|
|
}
|
|
else:
|
|
sts = boto3.client('sts',
|
|
region_name=region_name,
|
|
aws_access_key_id=aws_access_key_id,
|
|
aws_secret_access_key=aws_secret_access_key
|
|
)
|
|
sts_credential = sts.assume_role(
|
|
RoleArn=credential['role_arn'],
|
|
RoleSessionName='crawl-billing' + str(time.time()),
|
|
DurationSeconds=900,
|
|
ExternalId=credential['external_id']
|
|
)
|
|
return {
|
|
'aws_access_key_id': sts_credential['Credentials']['AccessKeyId'],
|
|
'aws_secret_access_key': sts_credential['Credentials']['SecretAccessKey'],
|
|
'region_name': region_name,
|
|
'aws_session_token': sts_credential['Credentials']['SessionToken'],
|
|
'bucket': credential['bucket']
|
|
}
|
|
|
|
|
|
import sys
|
|
|
|
if __name__ == '__main__':
|
|
# aws - billing - test - daily / aws - billing - test01 / 20230901 - 20231001 / aws - billing - test01 - 00001.
|
|
# csv.gz
|
|
# aws - billing - test - daily / aws - billing - test01 / 20230901 - 20231001 / aws - billing - test01 - 00002.
|
|
# csv.gz
|
|
filenames = (
|
|
# daimler
|
|
# '563347375094-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip',
|
|
# '563347375094-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip',
|
|
# '028328046529-aws-billing-test-daily/aws-billing-test01/20230901-20231001/028328046529-aws-billing-test01-00001.csv.gz',
|
|
# '028328046529-aws-billing-test01-00002.csv.gz',
|
|
# '028328046529-aws-billing-detailed-line-items-ACTS-2023-09.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-ACTS-Ningxia-2023-09.csv.zip',
|
|
# '563646727715-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip',
|
|
# '563646727715-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip',
|
|
# '565095721352-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip',
|
|
# '565095721352-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip',
|
|
# '565359735310-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip',
|
|
# '565359735310-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip',
|
|
# '010265887827-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip',
|
|
# '010265887827-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip',
|
|
# '011562250191-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip',
|
|
# '011562250191-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip',
|
|
# '011276597621-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip',
|
|
# '011276597621-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip'
|
|
# '047802167422-aws-cost-allocation-ACTS-2023-09.csv',
|
|
# '047802167422-aws-cost-allocation-ACTS-Ningxia-2023-09.csv',
|
|
# '047802167422-aws-cost-allocation-2023-09.csv'
|
|
|
|
'087353436373-aws-cost-allocation-ACTS-2024-06.csv'
|
|
'087353436373-aws-cost-allocation-ACTS-Ningxia-2024-03.csv',
|
|
'087353436373-aws-cost-allocation-2024-03.csv',
|
|
'087353436373-aws-cost-allocation-AWSMarketplace-2024-03.csv',
|
|
# 施耐德
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2022-08.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-04.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2022-07.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2022-07.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2022-02.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2022-02.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2022-03.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2022-03.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-2023-09.csv.zip',
|
|
# '028328046529-aws-billing-detailed-line-items-with-resources-and-tags-ACTS-Ningxia-2023-09.csv.zip',
|
|
)
|
|
# getBucket('435857483079')
|
|
# getFiles('435857483079')
|
|
# auto_download_file_form_s3(filenames)
|
|
#
|
|
# getBucket('028328046529')
|
|
getFiles('087353436373') |