haxala1r/cloud-computing-msc-ai-examples
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'dev_faafo' (#3) from dev_faafo into master

Browse Source 
Reviewed-on: https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/pulls/3
This commit is contained in:
srieger
2024-04-08 09:52:09 +02:00
parent fa712b19ab 5c06e7437b
commit 0c6c18ade4
19 changed files with 289 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
demo1-getting-started.py
View File

@@ -13,7 +13,7 @@ from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username, # Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions) # project etc., as coordinated in the lab sessions)
group_number = 30 group_number = X
######################################################################################################################## ########################################################################################################################
@@ -32,8 +32,8 @@ project_name = 'CloudComp' + str(group_number)
project_network = 'CloudComp' + str(group_number) + '-net' project_network = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance # The image to look for and use for the started instance
ubuntu_image_name = "Ubuntu 18.04 - Bionic Beaver - 64-bit - Cloud Based Image" #ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image"
# TODO: Ubuntu >18.04 would require major updates to faafo example again/better option: complete rewrite of example? ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image"
# default region # default region
region_name = 'RegionOne' region_name = 'RegionOne'
@@ -50,7 +50,7 @@ def main():
# make sure to include ex_force_auth_version='3.x_password', as needed in our environment # make sure to include ex_force_auth_version='3.x_password', as needed in our environment
provider = get_driver(Provider.OPENSTACK) provider = get_driver(Provider.OPENSTACK)
print("Opening connection to %s as %s..." % (auth_url, auth_username)) print(("Opening connection to %s as %s..." % (auth_url, auth_username)))
conn = provider(auth_username, conn = provider(auth_username,
auth_password, auth_password,

31
demo2-instance-with-init-script.py
View File

@@ -7,7 +7,7 @@ from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username, # Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions) # project etc., as coordinated in the lab sessions)
group_number = 30 group_number = X
# web service endpoint of the private cloud infrastructure # web service endpoint of the private cloud infrastructure
@@ -20,15 +20,16 @@ project_name = 'CloudComp' + str(group_number)
project_network = 'CloudComp' + str(group_number) + '-net' project_network = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance # The image to look for and use for the started instance
ubuntu_image_name = "Ubuntu 18.04 - Bionic Beaver - 64-bit - Cloud Based Image" #ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image"
# TODO: Ubuntu >18.04 would require major updates to faafo example again/better option: complete rewrite of example? ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image"
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key # The public key to be used for SSH connection, please make sure, that you have the corresponding private key
# #
# id_rsa.pub should look like this (standard sshd pubkey format): # id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
keypair_name = 'srieger-pub' #keypair_name = 'CloudComp30-keypair'
keypair_name = "srieger-pub"
pub_key_file = '~/.ssh/id_rsa.pub' pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 'm1.small' flavor_name = 'm1.small'
@@ -104,7 +105,7 @@ def main():
keypair_exists = True keypair_exists = True
if keypair_exists: if keypair_exists:
print('Keypair ' + keypair_name + ' already exists. Skipping import.') print(('Keypair ' + keypair_name + ' already exists. Skipping import.'))
else: else:
print('adding keypair...') print('adding keypair...')
conn.import_key_pair_from_file(keypair_name, pub_key_file) conn.import_key_pair_from_file(keypair_name, pub_key_file)
@@ -128,7 +129,7 @@ def main():
security_group_exists = True security_group_exists = True
if security_group_exists: if security_group_exists:
print('Security Group ' + all_in_one_security_group.name + ' already exists. Skipping creation.') print(('Security Group ' + all_in_one_security_group.name + ' already exists. Skipping creation.'))
else: else:
all_in_one_security_group = conn.ex_create_security_group(security_group_name, all_in_one_security_group = conn.ex_create_security_group(security_group_name,
'network access for all-in-one application.') 'network access for all-in-one application.')
@@ -144,8 +145,12 @@ def main():
# #
########################################################################### ###########################################################################
#hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh'
# testing / faafo dev branch:
hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/branch/dev_faafo/faafo/contrib/install.sh'
userdata = '''#!/usr/bin/env bash userdata = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i faafo -i messaging -r api -r worker -r demo -i faafo -i messaging -r api -r worker -r demo
''' '''
@@ -159,7 +164,7 @@ def main():
instance_exists = True instance_exists = True
if instance_exists: if instance_exists:
print('Instance ' + testing_instance.name + ' already exists. Skipping creation.') print(('Instance ' + testing_instance.name + ' already exists. Skipping creation.'))
exit() exit()
else: else:
print('Starting new all-in-one instance and wait until it is running...') print('Starting new all-in-one instance and wait until it is running...')
@@ -181,12 +186,12 @@ def main():
private_ip = None private_ip = None
if len(testing_instance.private_ips): if len(testing_instance.private_ips):
private_ip = testing_instance.private_ips[0] private_ip = testing_instance.private_ips[0]
print('Private IP found: {}'.format(private_ip)) print(('Private IP found: {}'.format(private_ip)))
public_ip = None public_ip = None
if len(testing_instance.public_ips): if len(testing_instance.public_ips):
public_ip = testing_instance.public_ips[0] public_ip = testing_instance.public_ips[0]
print('Public IP found: {}'.format(public_ip)) print(('Public IP found: {}'.format(public_ip)))
print('Checking for unused Floating IP...') print('Checking for unused Floating IP...')
unused_floating_ip = None unused_floating_ip = None
@@ -197,11 +202,11 @@ def main():
if not unused_floating_ip and len(conn.ex_list_floating_ip_pools()): if not unused_floating_ip and len(conn.ex_list_floating_ip_pools()):
pool = conn.ex_list_floating_ip_pools()[0] pool = conn.ex_list_floating_ip_pools()[0]
print('Allocating new Floating IP from pool: {}'.format(pool)) print(('Allocating new Floating IP from pool: {}'.format(pool)))
unused_floating_ip = pool.create_floating_ip() unused_floating_ip = pool.create_floating_ip()
if public_ip: if public_ip:
print('Instance ' + testing_instance.name + ' already has a public ip. Skipping attachment.') print(('Instance ' + testing_instance.name + ' already has a public ip. Skipping attachment.'))
elif unused_floating_ip: elif unused_floating_ip:
conn.ex_attach_floating_ip_to_node(testing_instance, unused_floating_ip) conn.ex_attach_floating_ip_to_node(testing_instance, unused_floating_ip)
@@ -214,7 +219,7 @@ def main():
actual_ip_address = private_ip actual_ip_address = private_ip
print('\n') print('\n')
print('The Fractals app will be deployed to http://{}\n'.format(actual_ip_address)) print(('The Fractals app will be deployed to http://{}\n'.format(actual_ip_address)))
print('You can use ssh to login to the instance using your private key. Default user name for official Ubuntu\n' print('You can use ssh to login to the instance using your private key. Default user name for official Ubuntu\n'
'Cloud Images is: ubuntu, so you can use, e.g.: "ssh -i ~/.ssh/id_rsa ubuntu@<floating-ip>" if your private\n' 'Cloud Images is: ubuntu, so you can use, e.g.: "ssh -i ~/.ssh/id_rsa ubuntu@<floating-ip>" if your private\n'

35
demo3-microservice.py
View File

@@ -11,7 +11,7 @@ from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username, # Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions) # project etc., as coordinated in the lab sessions)
group_number = 30 group_number = X
# web service endpoint of the private cloud infrastructure # web service endpoint of the private cloud infrastructure
@@ -24,15 +24,16 @@ project_name = 'CloudComp' + str(group_number)
project_network = 'CloudComp' + str(group_number) + '-net' project_network = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance # The image to look for and use for the started instance
ubuntu_image_name = "Ubuntu 18.04 - Bionic Beaver - 64-bit - Cloud Based Image" #ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image"
# TODO: Ubuntu >18.04 would require major updates to faafo example again/better option: complete rewrite of example? ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image"
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key # The public key to be used for SSH connection, please make sure, that you have the corresponding private key
# #
# id_rsa.pub should look like this (standard sshd pubkey format): # id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
keypair_name = 'srieger-pub' #keypair_name = 'CloudComp30-keypair'
keypair_name = "srieger-pub"
pub_key_file = '~/.ssh/id_rsa.pub' pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 'm1.small' flavor_name = 'm1.small'
@@ -109,7 +110,7 @@ def main():
keypair_exists = True keypair_exists = True
if keypair_exists: if keypair_exists:
print('Keypair ' + keypair_name + ' already exists. Skipping import.') print(('Keypair ' + keypair_name + ' already exists. Skipping import.'))
else: else:
print('adding keypair...') print('adding keypair...')
conn.import_key_pair_from_file(keypair_name, pub_key_file) conn.import_key_pair_from_file(keypair_name, pub_key_file)
@@ -133,7 +134,7 @@ def main():
security_group_exists = True security_group_exists = True
if security_group_exists: if security_group_exists:
print('Worker Security Group ' + worker_security_group.name + ' already exists. Skipping creation.') print(('Worker Security Group ' + worker_security_group.name + ' already exists. Skipping creation.'))
else: else:
worker_security_group = conn.ex_create_security_group('worker', 'for services that run on a worker node') worker_security_group = conn.ex_create_security_group('worker', 'for services that run on a worker node')
conn.ex_create_security_group_rule(worker_security_group, 'TCP', 22, 22) conn.ex_create_security_group_rule(worker_security_group, 'TCP', 22, 22)
@@ -148,7 +149,7 @@ def main():
security_group_exists = True security_group_exists = True
if security_group_exists: if security_group_exists:
print('Controller Security Group ' + controller_security_group.name + ' already exists. Skipping creation.') print(('Controller Security Group ' + controller_security_group.name + ' already exists. Skipping creation.'))
else: else:
controller_security_group = conn.ex_create_security_group('control', 'for services that run on a control node') controller_security_group = conn.ex_create_security_group('control', 'for services that run on a control node')
conn.ex_create_security_group_rule(controller_security_group, 'TCP', 22, 22) conn.ex_create_security_group_rule(controller_security_group, 'TCP', 22, 22)
@@ -165,14 +166,12 @@ def main():
# #
########################################################################### ###########################################################################
# https://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh #hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh'
# is currently broken, hence the "rabbitctl" lines were added in the example # testing / faafo dev branch:
# below, see also https://bugs.launchpad.net/faafo/+bug/1679710 hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/branch/dev_faafo/faafo/contrib/install.sh'
#
# Thanks to Stefan Friedmann for finding this fix ;)
userdata = '''#!/usr/bin/env bash userdata = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i messaging -i faafo -r api -i messaging -i faafo -r api
rabbitmqctl add_user faafo guest rabbitmqctl add_user faafo guest
rabbitmqctl set_user_tags faafo administrator rabbitmqctl set_user_tags faafo administrator
@@ -205,11 +204,11 @@ def main():
if not unused_floating_ip: if not unused_floating_ip:
pool = conn.ex_list_floating_ip_pools()[0] pool = conn.ex_list_floating_ip_pools()[0]
print('Allocating new Floating IP from pool: {}'.format(pool)) print(('Allocating new Floating IP from pool: {}'.format(pool)))
unused_floating_ip = pool.create_floating_ip() unused_floating_ip = pool.create_floating_ip()
conn.ex_attach_floating_ip_to_node(instance_controller_1, unused_floating_ip) conn.ex_attach_floating_ip_to_node(instance_controller_1, unused_floating_ip)
print('Controller Application will be deployed to http://%s' % unused_floating_ip.ip_address) print(('Controller Application will be deployed to http://%s' % unused_floating_ip.ip_address))
########################################################################### ###########################################################################
# #
@@ -231,7 +230,7 @@ def main():
########################################################################### ###########################################################################
userdata = '''#!/usr/bin/env bash userdata = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i faafo -r worker -e 'http://%(ip_controller)s' -m 'amqp://faafo:guest@%(ip_controller)s:5672/' -i faafo -r worker -e 'http://%(ip_controller)s' -m 'amqp://faafo:guest@%(ip_controller)s:5672/'
''' % {'ip_controller': ip_controller} ''' % {'ip_controller': ip_controller}
@@ -261,11 +260,11 @@ def main():
if not unused_floating_ip: if not unused_floating_ip:
pool = conn.ex_list_floating_ip_pools()[0] pool = conn.ex_list_floating_ip_pools()[0]
print('Allocating new Floating IP from pool: {}'.format(pool)) print(('Allocating new Floating IP from pool: {}'.format(pool)))
unused_floating_ip = pool.create_floating_ip() unused_floating_ip = pool.create_floating_ip()
conn.ex_attach_floating_ip_to_node(instance_worker_1, unused_floating_ip) conn.ex_attach_floating_ip_to_node(instance_worker_1, unused_floating_ip)
print('The worker will be available for SSH at %s' % unused_floating_ip.ip_address) print(('The worker will be available for SSH at %s' % unused_floating_ip.ip_address))
print('You can use ssh to login to the controller using your private key. After login, you can list available ' print('You can use ssh to login to the controller using your private key. After login, you can list available '
'fractals using "faafo list". To request the generation of new fractals, you can use "faafo create". ' 'fractals using "faafo list". To request the generation of new fractals, you can use "faafo create". '

28
demo4-scale-out-add-worker.py
View File

@@ -13,7 +13,7 @@ from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username, # Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions) # project etc., as coordinated in the lab sessions)
group_number = 30 group_number = X
# web service endpoint of the private cloud infrastructure # web service endpoint of the private cloud infrastructure
@@ -26,15 +26,16 @@ project_name = 'CloudComp' + str(group_number)
project_network = 'CloudComp' + str(group_number) + '-net' project_network = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance # The image to look for and use for the started instance
ubuntu_image_name = "Ubuntu 18.04 - Bionic Beaver - 64-bit - Cloud Based Image" #ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image"
# TODO: Ubuntu >18.04 would require major updates to faafo example again/better option: complete rewrite of example? ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image"
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key # The public key to be used for SSH connection, please make sure, that you have the corresponding private key
# #
# id_rsa.pub should look like this (standard sshd pubkey format): # id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
keypair_name = 'srieger-pub' #keypair_name = 'CloudComp30-keypair'
keypair_name = "srieger-pub"
pub_key_file = '~/.ssh/id_rsa.pub' pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 'm1.small' flavor_name = 'm1.small'
@@ -111,10 +112,10 @@ def main():
for instance in conn.list_nodes(): for instance in conn.list_nodes():
if instance.name == 'app-services': if instance.name == 'app-services':
services_ip = instance.private_ips[0] services_ip = instance.private_ips[0]
print('Found app-services fixed IP to be: ', services_ip) print(('Found app-services fixed IP to be: ', services_ip))
if instance.name == 'app-api-1': if instance.name == 'app-api-1':
api_1_ip = instance.private_ips[0] api_1_ip = instance.private_ips[0]
print('Found app-api-1 fixed IP to be: ', api_1_ip) print(('Found app-api-1 fixed IP to be: ', api_1_ip))
########################################################################### ###########################################################################
# #
@@ -129,7 +130,7 @@ def main():
keypair_exists = True keypair_exists = True
if keypair_exists: if keypair_exists:
print('Keypair ' + keypair_name + ' already exists. Skipping import.') print(('Keypair ' + keypair_name + ' already exists. Skipping import.'))
else: else:
print('adding keypair...') print('adding keypair...')
conn.import_key_pair_from_file(keypair_name, pub_key_file) conn.import_key_pair_from_file(keypair_name, pub_key_file)
@@ -145,10 +146,10 @@ def main():
def get_security_group(connection, security_group_name): def get_security_group(connection, security_group_name):
"""A helper function to check if security group already exists""" """A helper function to check if security group already exists"""
print('Checking for existing ' + security_group_name + ' security group...') print(('Checking for existing ' + security_group_name + ' security group...'))
for security_grp in connection.ex_list_security_groups(): for security_grp in connection.ex_list_security_groups():
if security_grp.name == security_group_name: if security_grp.name == security_group_name:
print('Security Group ' + security_group_name + ' already exists. Skipping creation.') print(('Security Group ' + security_group_name + ' already exists. Skipping creation.'))
return security_grp return security_grp
return False return False
@@ -167,13 +168,17 @@ def main():
# #
########################################################################### ###########################################################################
#hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh'
# testing / faafo dev branch:
hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/branch/dev_faafo/faafo/contrib/install.sh'
userdata_worker = '''#!/usr/bin/env bash userdata_worker = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/' -i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/'
''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip} ''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip}
# userdata-api-2 = '''#!/usr/bin/env bash # userdata-api-2 = '''#!/usr/bin/env bash
# curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ # curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
# -i faafo -r worker -e 'http://%(api_2_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/' # -i faafo -r worker -e 'http://%(api_2_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/'
# ''' % {'api_2_ip': api_2_ip, 'services_ip': services_ip} # ''' % {'api_2_ip': api_2_ip, 'services_ip': services_ip}
@@ -185,6 +190,7 @@ def main():
ex_userdata=userdata_worker, ex_userdata=userdata_worker,
ex_security_groups=[worker_security_group]) ex_security_groups=[worker_security_group])
print(instance_worker_3)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

38
demo4-scale-out.py
View File

@@ -13,7 +13,7 @@ from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username, # Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions) # project etc., as coordinated in the lab sessions)
group_number = 30 group_number = 2
# web service endpoint of the private cloud infrastructure # web service endpoint of the private cloud infrastructure
@@ -26,15 +26,17 @@ project_name = 'CloudComp' + str(group_number)
project_network = 'CloudComp' + str(group_number) + '-net' project_network = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance # The image to look for and use for the started instance
ubuntu_image_name = "Ubuntu 18.04 - Bionic Beaver - 64-bit - Cloud Based Image" #ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image"
# TODO: Ubuntu >18.04 would require major updates to faafo example again/better option: complete rewrite of example? ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image"
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key # The public key to be used for SSH connection, please make sure, that you have the corresponding private key
# #
# id_rsa.pub should look like this (standard sshd pubkey format): # id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
keypair_name = 'srieger-pub' #keypair_name = 'CloudComp30-keypair'
keypair_name = "srieger-pub"
pub_key_file = '~/.ssh/id_rsa.pub' pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 'm1.small' flavor_name = 'm1.small'
@@ -113,7 +115,7 @@ def main():
keypair_exists = True keypair_exists = True
if keypair_exists: if keypair_exists:
print('Keypair ' + keypair_name + ' already exists. Skipping import.') print(('Keypair ' + keypair_name + ' already exists. Skipping import.'))
else: else:
print('adding keypair...') print('adding keypair...')
conn.import_key_pair_from_file(keypair_name, pub_key_file) conn.import_key_pair_from_file(keypair_name, pub_key_file)
@@ -131,7 +133,7 @@ def main():
for instance in conn.list_nodes(): for instance in conn.list_nodes():
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller', if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller',
'app-services', 'app-api-1', 'app-api-2']: 'app-services', 'app-api-1', 'app-api-2']:
print('Destroying Instance: %s' % instance.name) print(('Destroying Instance: %s' % instance.name))
conn.destroy_node(instance) conn.destroy_node(instance)
# wait until all nodes are destroyed to be able to remove depended security groups # wait until all nodes are destroyed to be able to remove depended security groups
@@ -149,7 +151,7 @@ def main():
# delete security groups # delete security groups
for group in conn.ex_list_security_groups(): for group in conn.ex_list_security_groups():
if group.name in ['control', 'worker', 'api', 'services']: if group.name in ['control', 'worker', 'api', 'services']:
print('Deleting security group: %s' % group.name) print(('Deleting security group: %s' % group.name))
conn.ex_delete_security_group(group) conn.ex_delete_security_group(group)
########################################################################### ###########################################################################
@@ -160,10 +162,10 @@ def main():
def get_security_group(connection, security_group_name): def get_security_group(connection, security_group_name):
"""A helper function to check if security group already exists""" """A helper function to check if security group already exists"""
print('Checking for existing ' + security_group_name + ' security group...') print(('Checking for existing ' + security_group_name + ' security group...'))
for security_grp in connection.ex_list_security_groups(): for security_grp in connection.ex_list_security_groups():
if security_grp.name == security_group_name: if security_grp.name == security_group_name:
print('Security Group ' + security_group_name + ' already exists. Skipping creation.') print(('Security Group ' + security_group_name + ' already exists. Skipping creation.'))
return worker_security_group return worker_security_group
return False return False
@@ -226,14 +228,12 @@ def main():
# #
########################################################################### ###########################################################################
# https://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh #hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh'
# is currently broken, hence the "rabbitctl" lines were added in the example # testing / faafo dev branch:
# below, see also https://bugs.launchpad.net/faafo/+bug/1679710 hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/branch/dev_faafo/faafo/contrib/install.sh'
#
# Thanks to Stefan Friedmann for finding this fix ;)
userdata_service = '''#!/usr/bin/env bash userdata_service = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i database -i messaging -i database -i messaging
rabbitmqctl add_user faafo guest rabbitmqctl add_user faafo guest
rabbitmqctl set_user_tags faafo administrator rabbitmqctl set_user_tags faafo administrator
@@ -259,7 +259,7 @@ def main():
########################################################################### ###########################################################################
userdata_api = '''#!/usr/bin/env bash userdata_api = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i faafo -r api -m 'amqp://faafo:guest@%(services_ip)s:5672/' \ -i faafo -r api -m 'amqp://faafo:guest@%(services_ip)s:5672/' \
-d 'mysql+pymysql://faafo:password@%(services_ip)s:3306/faafo' -d 'mysql+pymysql://faafo:password@%(services_ip)s:3306/faafo'
''' % {'services_ip': services_ip} ''' % {'services_ip': services_ip}
@@ -292,7 +292,7 @@ def main():
for instance in [instance_api_1, instance_api_2]: for instance in [instance_api_1, instance_api_2]:
floating_ip = get_floating_ip(conn) floating_ip = get_floating_ip(conn)
conn.ex_attach_floating_ip_to_node(instance, floating_ip) conn.ex_attach_floating_ip_to_node(instance, floating_ip)
print('allocated %(ip)s to %(host)s' % {'ip': floating_ip.ip_address, 'host': instance.name}) print(('allocated %(ip)s to %(host)s' % {'ip': floating_ip.ip_address, 'host': instance.name}))
########################################################################### ###########################################################################
# #
@@ -301,12 +301,12 @@ def main():
########################################################################### ###########################################################################
userdata_worker = '''#!/usr/bin/env bash userdata_worker = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/' -i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/'
''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip} ''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip}
# userdata_api-api-2 = '''#!/usr/bin/env bash # userdata_api-api-2 = '''#!/usr/bin/env bash
# curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ # curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
# -i faafo -r worker -e 'http://%(api_2_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/' # -i faafo -r worker -e 'http://%(api_2_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/'
# ''' % {'api_2_ip': api_2_ip, 'services_ip': services_ip} # ''' % {'api_2_ip': api_2_ip, 'services_ip': services_ip}

6
demo5-1-durable-storage.py
View File

@@ -1,4 +1,4 @@
from __future__ import print_function
import getpass import getpass
import os import os
@@ -27,12 +27,12 @@ def main():
if "OS_PROJECT_NAME" in os.environ: if "OS_PROJECT_NAME" in os.environ:
project_name = os.environ["OS_PROJECT_NAME"] project_name = os.environ["OS_PROJECT_NAME"]
else: else:
project_name = input("Enter your OpenStack project:") project_name = eval(input("Enter your OpenStack project:"))
if "OS_USERNAME" in os.environ: if "OS_USERNAME" in os.environ:
auth_username = os.environ["OS_USERNAME"] auth_username = os.environ["OS_USERNAME"]
else: else:
auth_username = input("Enter your OpenStack username:") auth_username = eval(input("Enter your OpenStack username:"))
if "OS_PASSWORD" in os.environ: if "OS_PASSWORD" in os.environ:
auth_password = os.environ["OS_PASSWORD"] auth_password = os.environ["OS_PASSWORD"]

6
demo5-2-backup-fractals.py
View File

@@ -1,4 +1,4 @@
from __future__ import print_function
import getpass import getpass
import json import json
@@ -28,12 +28,12 @@ def main():
if "OS_PROJECT_NAME" in os.environ: if "OS_PROJECT_NAME" in os.environ:
project_name = os.environ["OS_PROJECT_NAME"] project_name = os.environ["OS_PROJECT_NAME"]
else: else:
project_name = input("Enter your OpenStack project:") project_name = eval(input("Enter your OpenStack project:"))
if "OS_USERNAME" in os.environ: if "OS_USERNAME" in os.environ:
auth_username = os.environ["OS_USERNAME"] auth_username = os.environ["OS_USERNAME"]
else: else:
auth_username = input("Enter your OpenStack username:") auth_username = eval(input("Enter your OpenStack username:"))
if "OS_PASSWORD" in os.environ: if "OS_PASSWORD" in os.environ:
auth_password = os.environ["OS_PASSWORD"] auth_password = os.environ["OS_PASSWORD"]

6
destroy-all-demo-instances.py
View File

@@ -13,7 +13,7 @@ from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username, # Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions) # project etc., as coordinated in the lab sessions)
group_number = 30 group_number = X
# web service endpoint of the private cloud infrastructure # web service endpoint of the private cloud infrastructure
@@ -70,7 +70,7 @@ def main():
for instance in conn.list_nodes(): for instance in conn.list_nodes():
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller', if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller',
'app-services', 'app-api-1', 'app-api-2']: 'app-services', 'app-api-1', 'app-api-2']:
print('Destroying Instance: %s' % instance.name) print(('Destroying Instance: %s' % instance.name))
conn.destroy_node(instance) conn.destroy_node(instance)
# wait until all nodes are destroyed to be able to remove depended security groups # wait until all nodes are destroyed to be able to remove depended security groups
@@ -89,7 +89,7 @@ def main():
# delete security groups # delete security groups
for group in conn.ex_list_security_groups(): for group in conn.ex_list_security_groups():
if group.name in ['control', 'worker', 'api', 'services']: if group.name in ['control', 'worker', 'api', 'services']:
print('Deleting security group: %s' % group.name) print(('Deleting security group: %s' % group.name))
conn.ex_delete_security_group(group) conn.ex_delete_security_group(group)

90
faafo/bin/faafo Normal file → Executable file
View File

@@ -12,7 +12,6 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import copy
import json import json
import random import random
import uuid import uuid
@@ -75,13 +74,19 @@ def get_random_task():
float(CONF.command.max_yb)) float(CONF.command.max_yb))
task = { task = {
'uuid': str(uuid.uuid4()), 'data': {
'width': width, 'type': 'fractal',
'height': height, 'attributes': {
'iterations': iterations, 'xa': xa, 'uuid': str(uuid.uuid4()),
'xb': xb, 'width': width,
'ya': ya, 'height': height,
'yb': yb 'iterations': iterations,
'xa': xa,
'xb': xb,
'ya': ya,
'yb': yb
}
}
} }
return task return task
@@ -93,25 +98,31 @@ def do_get_fractal():
def do_show_fractal(): def do_show_fractal():
LOG.info("showing fractal %s" % CONF.command.uuid) LOG.info("showing fractal %s" % CONF.command.uuid)
headers = {'Content-Type': 'application/vnd.api+json',
'Accept': 'application/vnd.api+json'}
result = requests.get("%s/v1/fractal/%s" % result = requests.get("%s/v1/fractal/%s" %
(CONF.endpoint_url, CONF.command.uuid)) (CONF.endpoint_url, CONF.command.uuid),
headers=headers)
LOG.debug("result: %s" % result.text)
if result.status_code == 200: if result.status_code == 200:
data = json.loads(result.text) data = json.loads(result.text)
fractal_data = data['data']['attributes']
output = PrettyTable(["Parameter", "Value"]) output = PrettyTable(["Parameter", "Value"])
output.align["Parameter"] = "l" output.align["Parameter"] = "l"
output.align["Value"] = "l" output.align["Value"] = "l"
output.add_row(["uuid", data['uuid']]) output.add_row(["uuid", fractal_data['uuid']])
output.add_row(["duration", "%f seconds" % data['duration']]) output.add_row(["duration", "%f seconds" % fractal_data['duration']])
output.add_row(["dimensions", "%d x %d pixels" % output.add_row(["dimensions", "%d x %d pixels" %
(data['width'], data['height'])]) (fractal_data['width'], fractal_data['height'])])
output.add_row(["iterations", data['iterations']]) output.add_row(["iterations", fractal_data['iterations']])
output.add_row(["xa", data['xa']]) output.add_row(["xa", fractal_data['xa']])
output.add_row(["xb", data['xb']]) output.add_row(["xb", fractal_data['xb']])
output.add_row(["ya", data['ya']]) output.add_row(["ya", fractal_data['ya']])
output.add_row(["yb", data['yb']]) output.add_row(["yb", fractal_data['yb']])
output.add_row(["size", "%d bytes" % data['size']]) output.add_row(["size", "%d bytes" % fractal_data['size']])
output.add_row(["checksum", data['checksum']]) output.add_row(["checksum", fractal_data['checksum']])
output.add_row(["generated_by", data['generated_by']]) output.add_row(["generated_by", fractal_data['generated_by']])
print(output) print(output)
else: else:
LOG.error("fractal '%s' not found" % CONF.command.uuid) LOG.error("fractal '%s' not found" % CONF.command.uuid)
@@ -123,34 +134,43 @@ def do_list_fractals():
fractals = get_fractals() fractals = get_fractals()
output = PrettyTable(["UUID", "Dimensions", "Filesize"]) output = PrettyTable(["UUID", "Dimensions", "Filesize"])
for fractal in fractals: for fractal in fractals:
fractal_data = fractal['attributes']
output.add_row([ output.add_row([
fractal["uuid"], fractal_data["uuid"],
"%d x %d pixels" % (fractal["width"], fractal["height"]), "%d x %d pixels" % (fractal_data["width"], fractal_data["height"]),
"%d bytes" % (fractal["size"] or 0), "%d bytes" % (fractal_data["size"] or 0),
]) ])
print(output) print(output)
def get_fractals(page=1): def get_fractals(page=1):
result = requests.get("%s/v1/fractal?page=%d" % headers = {'Content-Type': 'application/vnd.api+json',
(CONF.endpoint_url, page)) 'Accept': 'application/vnd.api+json'}
result = requests.get("%s/v1/fractal?page=%d&page[size]=10" %
(CONF.endpoint_url, page),
headers=headers)
LOG.debug("result: %s" % result.text)
fractals = [] fractals = []
if result.status_code == 200: if result.status_code == 200:
data = json.loads(result.text) data = json.loads(result.text)
if page < data['total_pages']: if (page * 10) < data['meta']['total']:
fractals = data['objects'] + get_fractals(page + 1) fractals = data['data'] + get_fractals(page + 1)
else: else:
return data['objects'] return data['data']
return fractals return fractals
def do_delete_fractal(): def do_delete_fractal():
LOG.info("deleting fractal %s" % CONF.command.uuid) LOG.info("deleting fractal %s" % CONF.command.uuid)
headers = {'Content-Type': 'application/vnd.api+json',
'Accept': 'application/vnd.api+json'}
result = requests.delete("%s/v1/fractal/%s" % result = requests.delete("%s/v1/fractal/%s" %
(CONF.endpoint_url, CONF.command.uuid)) (CONF.endpoint_url, CONF.command.uuid),
LOG.debug("result: %s" %result) headers=headers)
LOG.debug("result: %s" % result.text)
def do_create_fractal(): def do_create_fractal():
@@ -161,14 +181,14 @@ def do_create_fractal():
number = random.randint(int(CONF.command.min_tasks), number = random.randint(int(CONF.command.min_tasks),
int(CONF.command.max_tasks)) int(CONF.command.max_tasks))
LOG.info("generating %d task(s)" % number) LOG.info("generating %d task(s)" % number)
for i in xrange(0, number): for i in range(0, number):
task = get_random_task() task = get_random_task()
LOG.debug("created task %s" % task) LOG.debug("created task %s" % task)
# NOTE(berendt): only necessary when using requests < 2.4.2 headers = {'Content-Type': 'application/vnd.api+json',
headers = {'Content-type': 'application/json', 'Accept': 'application/vnd.api+json'}
'Accept': 'text/plain'} resp = requests.post("%s/v1/fractal" % CONF.endpoint_url,
requests.post("%s/v1/fractal" % CONF.endpoint_url,
json.dumps(task), headers=headers) json.dumps(task), headers=headers)
LOG.debug("resp: %s" % resp.text)
def add_command_parsers(subparsers): def add_command_parsers(subparsers):

11
faafo/contrib/install-aws.sh Normal file → Executable file
View File

@@ -78,6 +78,7 @@ if [[ -e /etc/os-release ]]; then
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
sudo apt-get update sudo apt-get update
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
sudo dnf update -y sudo dnf update -y
fi fi
@@ -89,6 +90,7 @@ if [[ -e /etc/os-release ]]; then
#sudo sed -i -e "/bind-address/d" /etc/mysql/my.cnf #sudo sed -i -e "/bind-address/d" /etc/mysql/my.cnf
sudo service mysql restart sudo service mysql restart
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
sudo dnf install -y mariadb-server python-mysql sudo dnf install -y mariadb-server python-mysql
printf "[mysqld]\nbind-address = 127.0.0.1\n" | sudo tee /etc/my.cnf.d/faafo.conf printf "[mysqld]\nbind-address = 127.0.0.1\n" | sudo tee /etc/my.cnf.d/faafo.conf
sudo systemctl enable mariadb sudo systemctl enable mariadb
@@ -105,7 +107,12 @@ if [[ -e /etc/os-release ]]; then
if [[ $INSTALL_MESSAGING -eq 1 ]]; then if [[ $INSTALL_MESSAGING -eq 1 ]]; then
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
sudo apt-get install -y rabbitmq-server sudo apt-get install -y rabbitmq-server
# fixes for rabbitmq setup
sudo rabbitmqctl add_user faafo guest
sudo rabbitmqctl set_user_tags faafo administrator
sudo rabbitmqctl set_permissions -p / faafo ".*" ".*" ".*"
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
sudo dnf install -y rabbitmq-server sudo dnf install -y rabbitmq-server
sudo systemctl enable rabbitmq-server sudo systemctl enable rabbitmq-server
sudo systemctl start rabbitmq-server sudo systemctl start rabbitmq-server
@@ -117,6 +124,7 @@ if [[ -e /etc/os-release ]]; then
if [[ $INSTALL_FAAFO -eq 1 ]]; then if [[ $INSTALL_FAAFO -eq 1 ]]; then
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
# TODO: needs to be updated for Ubuntu >= 20.04
sudo apt-get install -y python-dev python-pip supervisor git zlib1g-dev libmysqlclient-dev python-mysqldb sudo apt-get install -y python-dev python-pip supervisor git zlib1g-dev libmysqlclient-dev python-mysqldb
# Following is needed because of # Following is needed because of
# https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740 # https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740
@@ -131,6 +139,7 @@ if [[ -e /etc/os-release ]]; then
fi fi
fi fi
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
sudo dnf install -y python-devel python-pip supervisor git zlib-devel mariadb-devel gcc which python-mysql sudo dnf install -y python-devel python-pip supervisor git zlib-devel mariadb-devel gcc which python-mysql
sudo systemctl enable supervisord sudo systemctl enable supervisord
sudo systemctl start supervisord sudo systemctl start supervisord
@@ -171,6 +180,7 @@ startretries=0"
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
echo "$faafo_api" | sudo tee -a /etc/supervisor/conf.d/faafo.conf echo "$faafo_api" | sudo tee -a /etc/supervisor/conf.d/faafo.conf
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
echo "$faafo_api" | sudo tee -a /etc/supervisord.d/faafo.ini echo "$faafo_api" | sudo tee -a /etc/supervisord.d/faafo.ini
else else
echo "error: distribution $ID not supported" echo "error: distribution $ID not supported"
@@ -188,6 +198,7 @@ startretries=0"
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
echo "$faafo_worker" | sudo tee -a /etc/supervisor/conf.d/faafo.conf echo "$faafo_worker" | sudo tee -a /etc/supervisor/conf.d/faafo.conf
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
echo "$faafo_worker" | sudo tee -a /etc/supervisord.d/faafo.ini echo "$faafo_worker" | sudo tee -a /etc/supervisord.d/faafo.ini
else else
echo "error: distribution $ID not supported" echo "error: distribution $ID not supported"

33
faafo/contrib/install.sh Normal file → Executable file
View File

@@ -78,26 +78,30 @@ if [[ -e /etc/os-release ]]; then
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
sudo apt-get update sudo apt-get update
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
sudo dnf update -y sudo dnf update -y
fi fi
if [[ $INSTALL_DATABASE -eq 1 ]]; then if [[ $INSTALL_DATABASE -eq 1 ]]; then
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server python-mysqldb sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mariadb-server python3-mysqldb
# HSFD changes for Ubuntu 18.04 # HSFD changes for Ubuntu 18.04
sudo sed -i -e "/bind-address/d" /etc/mysql/mysql.conf.d/mysqld.cnf #sudo sed -i -e "/bind-address/d" /etc/mysql/mysql.conf.d/mysqld.cnf
#sudo sed -i -e "/bind-address/d" /etc/mysql/my.cnf ##sudo sed -i -e "/bind-address/d" /etc/mysql/my.cnf
sudo service mysql restart sudo sed -i -e "s/127.0.0.1/0.0.0.0/g" /etc/mysql/mariadb.conf.d/50-server.cnf
sudo mysqladmin password password
sudo systemctl restart mariadb
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
sudo dnf install -y mariadb-server python-mysql # fedora currently not tested nor supported
sudo dnf install -y mariadb-server python3-mysql
printf "[mysqld]\nbind-address = 127.0.0.1\n" | sudo tee /etc/my.cnf.d/faafo.conf printf "[mysqld]\nbind-address = 127.0.0.1\n" | sudo tee /etc/my.cnf.d/faafo.conf
sudo mysqladmin password password
sudo systemctl enable mariadb sudo systemctl enable mariadb
sudo systemctl start mariadb sudo systemctl start mariadb
else else
echo "error: distribution $ID not supported" echo "error: distribution $ID not supported"
exit 1 exit 1
fi fi
sudo mysqladmin password password
sudo mysql -uroot -ppassword mysql -e "CREATE DATABASE IF NOT EXISTS faafo; GRANT ALL PRIVILEGES ON faafo.* TO 'faafo'@'%' IDENTIFIED BY 'password';" sudo mysql -uroot -ppassword mysql -e "CREATE DATABASE IF NOT EXISTS faafo; GRANT ALL PRIVILEGES ON faafo.* TO 'faafo'@'%' IDENTIFIED BY 'password';"
URL_DATABASE='mysql://root:password@localhost/faafo' URL_DATABASE='mysql://root:password@localhost/faafo'
fi fi
@@ -105,7 +109,12 @@ if [[ -e /etc/os-release ]]; then
if [[ $INSTALL_MESSAGING -eq 1 ]]; then if [[ $INSTALL_MESSAGING -eq 1 ]]; then
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
sudo apt-get install -y rabbitmq-server sudo apt-get install -y rabbitmq-server
# fixes for rabbitmq setup
sudo rabbitmqctl add_user faafo guest
sudo rabbitmqctl set_user_tags faafo administrator
sudo rabbitmqctl set_permissions -p / faafo ".*" ".*" ".*"
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
sudo dnf install -y rabbitmq-server sudo dnf install -y rabbitmq-server
sudo systemctl enable rabbitmq-server sudo systemctl enable rabbitmq-server
sudo systemctl start rabbitmq-server sudo systemctl start rabbitmq-server
@@ -117,7 +126,7 @@ if [[ -e /etc/os-release ]]; then
if [[ $INSTALL_FAAFO -eq 1 ]]; then if [[ $INSTALL_FAAFO -eq 1 ]]; then
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
sudo apt-get install -y python-dev python-pip supervisor git zlib1g-dev libmysqlclient-dev python-mysqldb sudo apt-get install -y python3-dev python3-pip supervisor git zlib1g-dev libmysqlclient-dev python3-mysqldb python-is-python3
# Following is needed because of # Following is needed because of
# https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740 # https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740
if [ $(lsb_release --short --codename) = xenial ]; then if [ $(lsb_release --short --codename) = xenial ]; then
@@ -131,7 +140,8 @@ if [[ -e /etc/os-release ]]; then
fi fi
fi fi
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
sudo dnf install -y python-devel python-pip supervisor git zlib-devel mariadb-devel gcc which python-mysql # fedora currently not tested nor supported
sudo dnf install -y python3-devel python3-pip supervisor git zlib-devel mariadb-devel gcc which python3-mysql
sudo systemctl enable supervisord sudo systemctl enable supervisord
sudo systemctl start supervisord sudo systemctl start supervisord
#elif [[ $ID = 'opensuse' || $ID = 'sles' ]]; then #elif [[ $ID = 'opensuse' || $ID = 'sles' ]]; then
@@ -142,12 +152,13 @@ if [[ -e /etc/os-release ]]; then
fi fi
# HSFD changed to local repo # HSFD changed to local repo
git clone https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples #git clone https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples
git clone --branch dev_faafo https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples
cd cloud-computing-msc-ai-examples/faafo cd cloud-computing-msc-ai-examples/faafo
# following line required by bug 1636150 # following line required by bug 1636150
sudo pip install --upgrade pbr sudo pip install --upgrade pbr
sudo pip install -r requirements.txt sudo pip install -r requirements.txt
sudo python setup.py install sudo python3 setup.py install
sudo sed -i -e "s#transport_url = .*#transport_url = $URL_MESSAGING#" /etc/faafo/faafo.conf sudo sed -i -e "s#transport_url = .*#transport_url = $URL_MESSAGING#" /etc/faafo/faafo.conf
sudo sed -i -e "s#database_url = .*#database_url = $URL_DATABASE#" /etc/faafo/faafo.conf sudo sed -i -e "s#database_url = .*#database_url = $URL_DATABASE#" /etc/faafo/faafo.conf
@@ -165,6 +176,7 @@ startretries=0"
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
echo "$faafo_api" | sudo tee -a /etc/supervisor/conf.d/faafo.conf echo "$faafo_api" | sudo tee -a /etc/supervisor/conf.d/faafo.conf
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
echo "$faafo_api" | sudo tee -a /etc/supervisord.d/faafo.ini echo "$faafo_api" | sudo tee -a /etc/supervisord.d/faafo.ini
else else
echo "error: distribution $ID not supported" echo "error: distribution $ID not supported"
@@ -182,6 +194,7 @@ startretries=0"
if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then
echo "$faafo_worker" | sudo tee -a /etc/supervisor/conf.d/faafo.conf echo "$faafo_worker" | sudo tee -a /etc/supervisor/conf.d/faafo.conf
elif [[ $ID = 'fedora' ]]; then elif [[ $ID = 'fedora' ]]; then
# fedora currently not tested nor supported
echo "$faafo_worker" | sudo tee -a /etc/supervisord.d/faafo.ini echo "$faafo_worker" | sudo tee -a /etc/supervisord.d/faafo.ini
else else
echo "error: distribution $ID not supported" echo "error: distribution $ID not supported"

6
faafo/contrib/test_api.py
View File

@@ -34,11 +34,11 @@ assert response.status_code == 201
response = requests.get(url, headers=headers) response = requests.get(url, headers=headers)
assert response.status_code == 200 assert response.status_code == 200
print(response.json()) print((response.json()))
response = requests.get(url + '/' + uuid, headers=headers) response = requests.get(url + '/' + uuid, headers=headers)
assert response.status_code == 200 assert response.status_code == 200
print(response.json()) print((response.json()))
data = { data = {
'checksum': 'c6fef4ef13a577066c2281b53c82ce2c7e94e', 'checksum': 'c6fef4ef13a577066c2281b53c82ce2c7e94e',
@@ -50,7 +50,7 @@ assert response.status_code == 200
response = requests.get(url + '/' + uuid, headers=headers) response = requests.get(url + '/' + uuid, headers=headers)
assert response.status_code == 200 assert response.status_code == 200
print(response.json()) print((response.json()))
response = requests.delete(url + '/' + uuid, headers=headers) response = requests.delete(url + '/' + uuid, headers=headers)
assert response.status_code == 204 assert response.status_code == 204

4
faafo/doc/source/conf.py
View File

@@ -11,7 +11,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
copyright = u'2015, OpenStack contributors' copyright = '2015, OpenStack contributors'
master_doc = 'index' master_doc = 'index'
project = u'First App Application for OpenStack' project = 'First App Application for OpenStack'
source_suffix = '.rst' source_suffix = '.rst'

17
faafo/doc/source/conf.py.bak Normal file
View File

@@ -0,0 +1,17 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
copyright = u'2015, OpenStack contributors'
master_doc = 'index'
project = u'First App Application for OpenStack'
source_suffix = '.rst'

48
faafo/faafo/api/service.py
View File

@@ -12,7 +12,8 @@
import base64 import base64
import copy import copy
import cStringIO import io
import socket
from pkg_resources import resource_filename from pkg_resources import resource_filename
import flask import flask
@@ -60,7 +61,10 @@ template_path = resource_filename(__name__, "templates")
app = flask.Flask('faafo.api', template_folder=template_path) app = flask.Flask('faafo.api', template_folder=template_path)
app.config['DEBUG'] = CONF.debug app.config['DEBUG'] = CONF.debug
app.config['SQLALCHEMY_DATABASE_URI'] = CONF.database_url app.config['SQLALCHEMY_DATABASE_URI'] = CONF.database_url
db = SQLAlchemy(app)
with app.app_context():
db = SQLAlchemy(app)
Bootstrap(app) Bootstrap(app)
@@ -95,8 +99,10 @@ class Fractal(db.Model):
return '<Fractal %s>' % self.uuid return '<Fractal %s>' % self.uuid
db.create_all() with app.app_context():
manager = APIManager(app, flask_sqlalchemy_db=db) db.create_all()
manager = APIManager(app=app, session=db.session)
connection = Connection(CONF.transport_url) connection = Connection(CONF.transport_url)
@@ -104,10 +110,11 @@ connection = Connection(CONF.transport_url)
@app.route('/index', methods=['GET']) @app.route('/index', methods=['GET'])
@app.route('/index/<int:page>', methods=['GET']) @app.route('/index/<int:page>', methods=['GET'])
def index(page=1): def index(page=1):
hostname = socket.gethostname()
fractals = Fractal.query.filter( fractals = Fractal.query.filter(
(Fractal.checksum != None) & (Fractal.size != None)).paginate( # noqa (Fractal.checksum is not None) & (Fractal.size is not None)).paginate(
page, 5, error_out=False) page=page, per_page=5)
return flask.render_template('index.html', fractals=fractals) return flask.render_template('index.html', fractals=fractals, hostname=hostname)
@app.route('/fractal/<string:fractalid>', methods=['GET']) @app.route('/fractal/<string:fractalid>', methods=['GET'])
@@ -119,8 +126,8 @@ def get_fractal(fractalid):
response.status_code = 404 response.status_code = 404
else: else:
image_data = base64.b64decode(fractal.image) image_data = base64.b64decode(fractal.image)
image = Image.open(cStringIO.StringIO(image_data)) image = Image.open(io.BytesIO(image_data))
output = cStringIO.StringIO() output = io.BytesIO()
image.save(output, "PNG") image.save(output, "PNG")
image.seek(0) image.seek(0)
response = flask.make_response(output.getvalue()) response = flask.make_response(output.getvalue())
@@ -130,6 +137,7 @@ def get_fractal(fractalid):
def generate_fractal(**kwargs): def generate_fractal(**kwargs):
LOG.debug("Postprocessor called!" + str(kwargs))
with producers[connection].acquire(block=True) as producer: with producers[connection].acquire(block=True) as producer:
producer.publish(kwargs['result'], producer.publish(kwargs['result'],
serializer='json', serializer='json',
@@ -138,9 +146,21 @@ def generate_fractal(**kwargs):
routing_key='normal') routing_key='normal')
def convert_image_to_binary(**kwargs):
LOG.debug("Preprocessor call: " + str(kwargs))
if 'image' in kwargs['data']['data']['attributes']:
LOG.debug("Converting image to binary...")
kwargs['data']['data']['attributes']['image'] = \
str(kwargs['data']['data']['attributes']['image']).encode("ascii")
def main(): def main():
manager.create_api(Fractal, methods=['GET', 'POST', 'DELETE', 'PUT'], print("Starting API server - new...")
postprocessors={'POST': [generate_fractal]}, with app.app_context():
exclude_columns=['image'], manager.create_api(Fractal, methods=['GET', 'POST', 'DELETE', 'PATCH'],
url_prefix='/v1') postprocessors={'POST_RESOURCE': [generate_fractal]},
app.run(host=CONF.listen_address, port=CONF.bind_port) preprocessors={'PATCH_RESOURCE': [convert_image_to_binary]},
exclude=['image'],
url_prefix='/v1',
allow_client_generated_ids=True)
app.run(host=CONF.listen_address, port=CONF.bind_port, debug=True)

10
faafo/faafo/api/templates/index.html
View File

@@ -57,4 +57,14 @@ yb = {{ fractal.yb }}</pre>
</div> </div>
{% endfor %} {% endfor %}
{{render_pagination(fractals)}} {{render_pagination(fractals)}}
<!-- requires POST request to have Content-Type: "application/vnd.api+json" header, would require Fetch API or XMLHTTPRequest etc.
<div>
<form action="/v1/fractal" method="POST">
<input type="submit" value="Create new fractals">
</form>
</div>
-->
<div>
Rendered by server: {{hostname}}
</div>
{% endblock %} {% endblock %}

38
faafo/faafo/worker/service.py
View File

@@ -19,12 +19,13 @@ import copy
import hashlib import hashlib
import json import json
import os import os
from PIL import Image
import random import random
import socket import socket
import tempfile import tempfile
import time import time
from PIL import Image
from kombu.mixins import ConsumerMixin from kombu.mixins import ConsumerMixin
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log from oslo_log import log
@@ -110,7 +111,8 @@ class Worker(ConsumerMixin):
accept=['json'], accept=['json'],
callbacks=[self.process])] callbacks=[self.process])]
def process(self, task, message): def process(self, task_def, message):
task = task_def['data']['attributes']
LOG.info("processing task %s" % task['uuid']) LOG.info("processing task %s" % task['uuid'])
LOG.debug(task) LOG.debug(task)
start_time = time.time() start_time = time.time()
@@ -136,21 +138,29 @@ class Worker(ConsumerMixin):
LOG.debug("removed temporary file %s" % filename) LOG.debug("removed temporary file %s" % filename)
result = { result = {
'uuid': task['uuid'], 'data': {
'duration': elapsed_time, 'type': 'fractal',
'image': image, 'id': task['uuid'],
'checksum': checksum, 'attributes': {
'size': size, 'uuid': task['uuid'],
'generated_by': socket.gethostname() 'duration': elapsed_time,
'image': image.decode("ascii"),
'checksum': checksum,
'size': size,
'generated_by': socket.gethostname()
}
}
} }
# NOTE(berendt): only necessary when using requests < 2.4.2 headers = {'Content-Type': 'application/vnd.api+json',
headers = {'Content-type': 'application/json', 'Accept': 'application/vnd.api+json'}
'Accept': 'text/plain'}
requests.put("%s/v1/fractal/%s" % resp = requests.patch("%s/v1/fractal/%s" %
(CONF.endpoint_url, str(task['uuid'])), (CONF.endpoint_url, str(task['uuid'])),
json.dumps(result), headers=headers) json.dumps(result),
headers=headers,
timeout=30)
LOG.debug("Result: %s" % resp.text)
message.ack() message.ack()
return result return result

37
faafo/requirements.txt
View File

@@ -1,18 +1,31 @@
pbr>=1.6 #pbr>=1.6
pbr
pytz pytz
positional positional
iso8601 iso8601
anyjson>=0.3.3 #anyjson>=0.3.3
eventlet>=0.17.4 #anyjson
PyMySQL>=0.6.2,<0.7 # 0.7 design change breaks faafo, MIT License #eventlet>=0.17.4
SQLAlchemy>1.3,<1.4 # 1.4 breaks faafo list eventlet
Pillow==2.4.0 # MIT #PyMySQL>=0.6.2,<0.7 # 0.7 design change breaks faafo, MIT License
requests>=2.5.2 #PyMySQL>=0.6.2
PyMySQL
#SQLAlchemy>1.3,<1.4 # 1.4 breaks faafo list
#SQLAlchemy>=2.0.16
SQLAlchemy
#Pillow==2.4.0 # MIT
Pillow
#requests>=2.5.2
requests
Flask-Bootstrap Flask-Bootstrap
Flask Flask
flask-restless flask-restless-ng
flask-sqlalchemy flask-sqlalchemy
oslo.config>=2.3.0 # Apache-2.0 #oslo.config>=2.3.0 # Apache-2.0
oslo.log>=1.8.0 # Apache-2.0 #oslo.log>=1.8.0 # Apache-2.0
PrettyTable>=0.7,<0.8 #PrettyTable>=0.7,<0.8
kombu>=3.0.7 #kombu>=3.0.7
oslo.config
oslo.log
PrettyTable
kombu

3
faafo/setup.cfg
View File

@@ -13,8 +13,7 @@ classifier =
License :: OSI Approved :: Apache Software License License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux Operating System :: POSIX :: Linux
Programming Language :: Python Programming Language :: Python
Programming Language :: Python :: 2 Programming Language :: Python :: 3
Programming Language :: Python :: 2.7
[files] [files]
packages = packages =