141 lines
4.1 KiB
Python
141 lines
4.1 KiB
Python
|
"""
|
||
|
|
Functions for MiroSite setup
|
||
|
|
"""
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import tarfile
|
||
|
|
import tempfile
|
||
|
|
import os
|
||
|
|
import mimetypes
|
||
|
|
|
||
|
|
import boto3
|
||
|
|
from jinja2 import Environment, FileSystemLoader
|
||
|
|
|
||
|
|
|
||
|
|
def upload_website(bucket_name, website_dir, aws_access_key, aws_secret_key):
|
||
|
|
"""
|
||
|
|
upload_website
|
||
|
|
|
||
|
|
:param bucket_name: The name of the bucket to work with
|
||
|
|
:param website_dir: Folder of website files
|
||
|
|
:param aws_access_key: Authentication access key
|
||
|
|
:param aws_secret_key: Authentication secret key
|
||
|
|
|
||
|
|
:return
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Create client connection
|
||
|
|
s3 = boto3.client('s3',
|
||
|
|
aws_access_key_id=aws_access_key,
|
||
|
|
aws_secret_access_key=aws_secret_key)
|
||
|
|
|
||
|
|
# Loop all the files
|
||
|
|
for root, _, files in os.walk(website_dir):
|
||
|
|
for file in files:
|
||
|
|
try:
|
||
|
|
local_path = os.path.join(root, file)
|
||
|
|
s3_path = os.path.relpath(local_path, website_dir)
|
||
|
|
|
||
|
|
# Determine the Content-Type based on file extension
|
||
|
|
content_type, _ = mimetypes.guess_type(local_path)
|
||
|
|
content_type = content_type or 'application/octet-stream'
|
||
|
|
|
||
|
|
# Upload file to S3
|
||
|
|
s3.upload_file(local_path,
|
||
|
|
bucket_name,
|
||
|
|
s3_path,
|
||
|
|
ExtraArgs={'ContentType': content_type})
|
||
|
|
except Exception as ex:
|
||
|
|
logging.error("[FILE UPLOAD] [%s] [%s] [%s]", bucket_name, file, str(ex))
|
||
|
|
|
||
|
|
|
||
|
|
def download_website_gzip(bucket_name, region_name, website_gzip_name,
|
||
|
|
aws_access_key, aws_secret_key):
|
||
|
|
"""
|
||
|
|
download_website_gzip
|
||
|
|
|
||
|
|
:param bucket_name: The name of the bucket to work with
|
||
|
|
:param region_name: The region of the bucket
|
||
|
|
:param website_gzip_name: Name of file to download
|
||
|
|
:param aws_access_key: Authentication access key
|
||
|
|
:param aws_secret_key: Authentication secret key
|
||
|
|
|
||
|
|
:return Boolean for success
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Create client connection
|
||
|
|
s3 = boto3.client('s3',
|
||
|
|
endpoint_url=f'https://{region_name}.linodeobjects.com',
|
||
|
|
aws_access_key_id=aws_access_key,
|
||
|
|
aws_secret_access_key=aws_secret_key)
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Download file from S3
|
||
|
|
s3.download_file(bucket_name,
|
||
|
|
website_gzip_name,
|
||
|
|
website_gzip_name)
|
||
|
|
return True
|
||
|
|
except Exception as ex:
|
||
|
|
logging.error("[WEBSITE GZIP DOWNLOAD] [%s] [%s] [%s]", bucket_name,
|
||
|
|
website_gzip_name,
|
||
|
|
str(ex))
|
||
|
|
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def extract_website(website_gzip):
|
||
|
|
"""
|
||
|
|
extract_website
|
||
|
|
|
||
|
|
:param website_gzip:
|
||
|
|
|
||
|
|
:return Folder of extracted website
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Create a temporary directory
|
||
|
|
extract_path = tempfile.mkdtemp()
|
||
|
|
|
||
|
|
with tarfile.open(website_gzip, 'r:gz') as tar:
|
||
|
|
tar.extractall(extract_path)
|
||
|
|
|
||
|
|
return extract_path
|
||
|
|
|
||
|
|
|
||
|
|
def apply_template_website(website_dir, template_dict):
|
||
|
|
"""
|
||
|
|
apply_template_website
|
||
|
|
|
||
|
|
:param website_dir: Folder of website files
|
||
|
|
:param template_dict: Template dict
|
||
|
|
|
||
|
|
:return
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Setup skip dirs
|
||
|
|
skip_dirs = [os.path.join(website_dir, "img")]
|
||
|
|
|
||
|
|
# Loop all the files
|
||
|
|
for root, _, files in os.walk(website_dir):
|
||
|
|
# Skip certain dirs
|
||
|
|
if root in skip_dirs:
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Setup jinja2 loader
|
||
|
|
env = Environment(loader=FileSystemLoader(root))
|
||
|
|
|
||
|
|
# Loop all files in directory
|
||
|
|
for file in files:
|
||
|
|
try:
|
||
|
|
# Build file path
|
||
|
|
local_path = os.path.join(root, file)
|
||
|
|
|
||
|
|
# Get jinja2 template
|
||
|
|
template = env.get_template(file)
|
||
|
|
rendered_content = template.render(template_data=template_dict)
|
||
|
|
|
||
|
|
# Write the rendered file
|
||
|
|
with open(local_path, 'w', encoding='utf-8') as outfile:
|
||
|
|
outfile.write(rendered_content)
|
||
|
|
except Exception as ex:
|
||
|
|
logging.error("[FILE JINJA] [%s] [%s]", file, str(ex))
|