Skip to content

Issue #906 add support for $AWS_PROFILE #966

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Jun 3, 2018
Merged

Issue #906 add support for $AWS_PROFILE #966

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
66d9ff8
Issue #906 add support for $AWS_PROFILE
taraspos Apr 28, 2018
c49d497
make it work with python2
taraspos Apr 28, 2018
9c93747
replace `Config()` with `self`
taraspos Apr 30, 2018
eda474d
Merge branch 'master' of https://github.com/s3tools/s3cmd
taraspos May 7, 2018
406608d
Merge branch 'master' of https://github.com/Trane9991/s3cmd
taraspos May 7, 2018
e384565
Completed some PR tasks:- better imports handling- more debug logging…
taraspos May 7, 2018
a2065f4
pr tasks:
taraspos May 8, 2018
077027c
add helper function to read the deprecated credentials format
taraspos May 8, 2018
0483b54
add more comments
taraspos May 8, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 74 additions & 32 deletions S3/Config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@
import http.client as httplib
import locale

try:
from configparser import NoOptionError, NoSectionError, MissingSectionHeaderError, ConfigParser as PyConfigParser
except ImportError:
# Python2 fallback code
from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError, ConfigParser as PyConfigParser

try:
unicode
except NameError:
Expand Down Expand Up @@ -211,8 +217,8 @@ def __init__(self, configfile = None, access_key=None, secret_key=None, access_t
try:
self.read_config_file(configfile)
except IOError:
if 'AWS_CREDENTIAL_FILE' in os.environ:
self.env_config()
if 'AWS_CREDENTIAL_FILE' in os.environ or 'AWS_PROFILE' in os.environ:
self.aws_credential_file()

# override these if passed on the command-line
if access_key and secret_key:
Expand Down Expand Up @@ -275,38 +281,74 @@ def role_refresh(self):
except:
warning("Could not refresh role")

def env_config(self):
cred_content = ""
def aws_credential_file(self):
try:
cred_file = open(os.environ['AWS_CREDENTIAL_FILE'],'r')
cred_content = cred_file.read()
aws_credential_file = os.path.expanduser('~/.aws/credentials')
if 'AWS_CREDENTIAL_FILE' in os.environ and os.path.isfile(os.environ['AWS_CREDENTIAL_FILE']):
aws_credential_file = config_unicodise(os.environ['AWS_CREDENTIAL_FILE'])

config = PyConfigParser()

debug("Reading AWS credentials from %s" % (aws_credential_file))
try:
config.read(aws_credential_file)
except MissingSectionHeaderError:
# if header is missing, this could be deprecated credentials file format
# as described here: https://blog.csanchez.org/2011/05/
# then do the hacky-hack and add default header
# to be able to read the file with PyConfigParser()
config_string = None
with open(aws_credential_file, 'r') as f:
config_string = '[default]\n' + f.read()
config.read_string(config_string.decode('utf-8'))


profile = config_unicodise(os.environ.get('AWS_PROFILE', "default"))
debug("Using AWS profile '%s'" % (profile))

# get_key - helper function to read the aws profile credentials
# including the legacy ones as described here: https://blog.csanchez.org/2011/05/
def get_key(profile, key, legacy_key, print_warning=True):
result = None

try:
result = config.get(profile, key)
except NoOptionError as e:
if print_warning: # we may want to skip warning message for optional keys
warning("Couldn't find key '%s' for the AWS Profile '%s' in the credentials file '%s'" % (e.option, e.section, aws_credential_file))
if legacy_key: # if the legacy_key defined and original one wasn't found, try read the legacy_key
try:
key = legacy_key
profile = "default"
result = config.get(profile, key)
warning(
"Legacy configuratin key '%s' used, " % (key) +
"please use the standardized config format as described here: " +
"https://aws.amazon.com/blogs/security/a-new-and-standardized-way-to-manage-credentials-in-the-aws-sdks/"
)
except NoOptionError as e:
pass

if result:
debug("Found the configuration option '%s' for the AWS Profile '%s' in the credentials file %s" % (key, profile, aws_credential_file))
return result

profile_access_key = get_key(profile, "aws_access_key_id", "AWSAccessKeyId")
if profile_access_key:
Config().update_option('access_key', config_unicodise(profile_access_key))

profile_secret_key = get_key(profile, "aws_secret_access_key", "AWSSecretKey")
if profile_secret_key:
Config().update_option('secret_key', config_unicodise(profile_secret_key))

profile_access_token = get_key(profile, "aws_session_token", None, False)
if profile_access_token:
Config().update_option('access_token', config_unicodise(profile_access_token))

except IOError as e:
debug("Error %d accessing credentials file %s" % (e.errno,os.environ['AWS_CREDENTIAL_FILE']))
r_data = re.compile("^\s*(?P<orig_key>\w+)\s*=\s*(?P<value>.*)")
r_quotes = re.compile("^\"(.*)\"\s*$")
if len(cred_content)>0:
for line in cred_content.splitlines():
is_data = r_data.match(line)
if is_data:
data = is_data.groupdict()
if r_quotes.match(data["value"]):
data["value"] = data["value"][1:-1]
if data["orig_key"] == "AWSAccessKeyId" \
or data["orig_key"] == "aws_access_key_id":
data["key"] = "access_key"
elif data["orig_key"]=="AWSSecretKey" \
Copy link
Contributor

@fviard fviard May 7, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have found a reference about that:
https://blog.csanchez.org/2011/05/
Looks like it used to be the case up to 2011 at least.
Its a little bit annoying to have to take care of that, but that would probably be better to still handle that case.
Note: a warning might still be issued.
But users could have inherited config files or found old tutorial on internet and still be using that.
It is bad to break user existing setups just for updates when it is not strictly necessary.

or data["orig_key"]=="aws_secret_access_key":
data["key"] = "secret_key"
else:
debug("env_config: key = %r will be ignored", data["orig_key"])

if "key" in data:
Config().update_option(data["key"], data["value"])
if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):
print_value = ("%s...%d_chars...%s") % (data["value"][:2], len(data["value"]) - 3, data["value"][-1:])
else:
print_value = data["value"]
debug("env_Config: %s->%s" % (data["key"], print_value))
warning("%d accessing credentials file %s" % (e.errno, aws_credential_file))
except NoSectionError as e:
warning("Couldn't find AWS Profile '%s' in the credentials file '%s'" % (profile, aws_credential_file))

def option_list(self):
retval = []
Expand Down