haxala1r/cloud-computing-msc-ai-examples
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'dev_faafo' (#4) from dev_faafo into master

Browse Source 
Reviewed-on: https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/pulls/4
This commit is contained in:
srieger
2024-04-12 14:12:42 +02:00
parent 0c6c18ade4 c67d8886d7
commit e63fa984f3
7 changed files with 411 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
CloudComp-openrc Normal file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# To use an OpenStack cloud you need to authenticate against the Identity
# service named keystone, which returns a **Token** and **Service Catalog**.
# The catalog contains the endpoints for all services the user/tenant has
# access to - such as Compute, Image Service, Identity, Object Storage, Block
# Storage, and Networking (code-named nova, glance, keystone, swift,
# cinder, and neutron).
# unset all OPENSTACK ENV VARS
unset $(env | grep OS_ | cut -d "=" -f 1)
export OS_CACERT=./root-ca.crt
export GROUP_NUMBER=0
#
# *NOTE*: Using the 3 *Identity API* does not necessarily mean any other
# OpenStack API is version 3. For example, your cloud provider may implement
# Image API v1.1, Block Storage API v2, and Compute API v2.0. OS_AUTH_URL is
# only for the Identity API served through keystone.
export OS_AUTH_URL=https://10.32.4.182:5000/v3
# With the addition of Keystone we have standardized on the term **project**
# as the entity that owns the resources.
#export OS_PROJECT_ID=bba62cf6bf0b447491829d207e1b05f9
export OS_PROJECT_NAME="CloudComp$GROUP_NUMBER"
unset OS_DOMAIN_NAME
export OS_USER_DOMAIN_NAME="Default"
if [ -z "$OS_USER_DOMAIN_NAME" ]; then unset OS_USER_DOMAIN_NAME; fi
export OS_PROJECT_DOMAIN_ID="default"
if [ -z "$OS_PROJECT_DOMAIN_ID" ]; then unset OS_PROJECT_DOMAIN_ID; fi
# unset v2.0 items in case set
unset OS_TENANT_ID
unset OS_TENANT_NAME
# In addition to the owning entity (tenant), OpenStack stores the entity
# performing the action as the **user**.
export OS_USERNAME="CloudComp$GROUP_NUMBER"
# With Keystone you pass the keystone password.
#echo "Please enter your OpenStack Password for project $OS_PROJECT_NAME as user $OS_USERNAME: "
#read -sr OS_PASSWORD_INPUT
#export OS_PASSWORD=$OS_PASSWORD_INPUT
export OS_PASSWORD="demo"
# If your configuration has multiple regions, we set that information here.
# OS_REGION_NAME is optional and only valid in certain environments.
export OS_REGION_NAME="RegionOne"
# Don't leave a blank variable, unset it if it was empty
if [ -z "$OS_REGION_NAME" ]; then unset OS_REGION_NAME; fi
export OS_INTERFACE=public
export OS_IDENTITY_API_VERSION=3

70
demo1-getting-started.py → demo1_getting_started.py
View File

@@ -1,47 +1,64 @@
# Example for Cloud Computing Course Master AI / GSD """Example for Cloud Computing Course Master AI / GSD"""
# This script demonstrates how to use libcloud to start an instance in an OpenStack environment.
# The script will start an instance, list all instances, and then destroy the instance again.
# #
# uses libCloud: https://libcloud.apache.org/ # The script uses the libcloud library to interact with the OpenStack API.
# Need to install libcloud first: pip install apache-libcloud
#
# libCloud: https://libcloud.apache.org/
# libCloud API documentation: https://libcloud.readthedocs.io/en/latest/ # libCloud API documentation: https://libcloud.readthedocs.io/en/latest/
# OpenStack API documentation: https://developer.openstack.org/ # OpenStack API documentation: https://developer.openstack.org/
# this code was initially based on the former tutorial: https://developer.openstack.org/firstapp-libcloud/ # this code was initially based on the former tutorial:
# https://developer.openstack.org/firstapp-libcloud/
import getpass # Only needed for the password prompt:
# import getpass
from libcloud.compute.providers import get_driver from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username, # For our new Charmed OpenStack private cloud, we need to specify the path to the
# project etc., as coordinated in the lab sessions) # root CA certificate
import libcloud.security
libcloud.security.CA_CERTS_PATH = ['./root-ca.crt']
# Disable SSL certificate verification (not recommended for production)
# libcloud.security.VERIFY_SSL_CERT = False
group_number = X # Please use 1-29 for 0 in the following variable to specify your group number.
# (will be used for the username, project etc., as coordinated in the lab sessions)
GROUP_NUMBER = 0
######################################################################################################################## ###############################################################################################
# #
# no changes necessary below this line in this example # no changes necessary below this line in this example
# #
######################################################################################################################## ###############################################################################################
# web service endpoint of the private cloud infrastructure # web service endpoint of the private cloud infrastructure
auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000' # auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000'
AUTH_URL = 'https://10.32.4.182:5000'
# auth_url = 'https://private-cloud2.informatik.hs-fulda.de:5000'
# your username in OpenStack # your username in OpenStack
auth_username = 'CloudComp' + str(group_number) AUTH_USERNAME = 'CloudComp' + str(GROUP_NUMBER)
# your project in OpenStack # your project in OpenStack
project_name = 'CloudComp' + str(group_number) PROJECT_NAME = 'CloudComp' + str(GROUP_NUMBER)
# A network in the project the started instance will be attached to # A network in the project the started instance will be attached to
project_network = 'CloudComp' + str(group_number) + '-net' PROJECT_NETWORK = 'CloudComp' + str(GROUP_NUMBER) + '-net'
# The image to look for and use for the started instance # The image to look for and use for the started instance
#ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image" # ubuntu_image_name = "Ubuntu 18.04 - Bionic Beaver - 64-bit - Cloud Based Image"
ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image" UBUNTU_IMAGE_NAME = "auto-sync/ubuntu-jammy-22.04-amd64-server-20240319-disk1.img"
# default region # default region
region_name = 'RegionOne' REGION_NAME = 'RegionOne'
# domain to use, "default" for local accounts, "hsfulda" for RZ LDAP, e.g., using fdaiXXXX as auth_username # domain to use, "default" for local accounts, formerly "hsfulda" for LDAP accounts etc.
# domain_name = "default" # domain_name = "default"
def main(): def main(): # noqa: C901 pylint: disable=too-many-branches,too-many-statements,too-many-locals,missing-function-docstring
# get the password from user # get the password from user
# auth_password = getpass.getpass("Enter your OpenStack password:") # auth_password = getpass.getpass("Enter your OpenStack password:")
auth_password = "demo" auth_password = "demo"
@@ -50,14 +67,14 @@ def main():
# make sure to include ex_force_auth_version='3.x_password', as needed in our environment # make sure to include ex_force_auth_version='3.x_password', as needed in our environment
provider = get_driver(Provider.OPENSTACK) provider = get_driver(Provider.OPENSTACK)
print(("Opening connection to %s as %s..." % (auth_url, auth_username))) print(f"Opening connection to {AUTH_URL} as {AUTH_USERNAME}...")
conn = provider(auth_username, conn = provider(AUTH_USERNAME,
auth_password, auth_password,
ex_force_auth_url=auth_url, ex_force_auth_url=AUTH_URL,
ex_force_auth_version='3.x_password', ex_force_auth_version='3.x_password',
ex_tenant_name=project_name, ex_tenant_name=PROJECT_NAME,
ex_force_service_region=region_name) ex_force_service_region=REGION_NAME)
# ex_domain_name=domain_name) # ex_domain_name=domain_name)
print("Getting images and selecting desired one...") print("Getting images and selecting desired one...")
@@ -67,7 +84,7 @@ def main():
images = conn.list_images() images = conn.list_images()
image = '' image = ''
for img in images: for img in images:
if img.name == ubuntu_image_name: if img.name == UBUNTU_IMAGE_NAME:
image = img image = img
print(img) print(img)
@@ -94,7 +111,7 @@ def main():
networks = conn.ex_list_networks() networks = conn.ex_list_networks()
network = '' network = ''
for net in networks: for net in networks:
if net.name == project_network: if net.name == PROJECT_NETWORK:
network = net network = net
print("Create instance 'testing'...") print("Create instance 'testing'...")
@@ -103,7 +120,8 @@ def main():
# create a new instance with the name "testing" # create a new instance with the name "testing"
# make sure to provide networks (networks={network}) the instance should be attached to # make sure to provide networks (networks={network}) the instance should be attached to
instance_name = 'testing' instance_name = 'testing'
testing_instance = conn.create_node(name=instance_name, image=image, size=flavor, networks={network}) testing_instance = conn.create_node(name=instance_name, image=image, size=flavor,
networks={network})
print(testing_instance) print(testing_instance)
print("Showing all running instances...") print("Showing all running instances...")

131
demo2-instance-with-init-script.py → demo2_instance_with_init_script.py
View File

@@ -1,47 +1,75 @@
"""Example for Cloud Computing Course Master AI / GSD"""
# This script demonstrates how to use libcloud to start an instance in an OpenStack environment.
# The script will create and install a new SSH key pair, create a security group, start an instance
# and deploy a demo app (faafo) using cloud-init and assign a floating IP to the instance.
#
# cloud-init is a multi-distribution package that handles early initialization of a cloud instance.
# It is supported by many major cloud providers, including OpenStack.
# cloud-init documentation: https://cloudinit.readthedocs.io/en/latest/
# Needed if the password should be prompted for:
# import getpass # import getpass
# import os import os
import sys
from libcloud.compute.providers import get_driver from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username, # For our new Charmed OpenStack private cloud, we need to specify the path to the root
# project etc., as coordinated in the lab sessions) # CA certificate
import libcloud.security
libcloud.security.CA_CERTS_PATH = ['./root-ca.crt']
# Disable SSL certificate verification (not recommended for production)
# libcloud.security.VERIFY_SSL_CERT = False
group_number = X # Please use 1-29 as environment variable GROUP_NUMBER to specify your group number.
# (will be used for the username, project etc., as coordinated in the lab sessions)
group_number = os.environ.get('GROUP_NUMBER')
if group_number is None:
sys.exit('Please set the GROUP_NUMBER environment variable to your group number,\n'
'e.g., on Windows:\n'
' "$env:GROUP_NUMBER=0" or "set GROUP_NUMBER=0"\n'
'or on Linux/MacOS:\n'
' "export GROUP_NUMBER=0" or "set GROUP_NUMBER=0"')
# web service endpoint of the private cloud infrastructure # web service endpoint of the private cloud infrastructure
auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000' # auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000'
AUTH_URL = 'https://10.32.4.182:5000'
# auth_url = 'https://private-cloud2.informatik.hs-fulda.de:5000'
# your username in OpenStack # your username in OpenStack
auth_username = 'CloudComp' + str(group_number) AUTH_USERNAME = 'CloudComp' + str(group_number)
print(f'Using username: {AUTH_USERNAME}\n')
# your project in OpenStack # your project in OpenStack
project_name = 'CloudComp' + str(group_number) PROJECT_NAME = 'CloudComp' + str(group_number)
# A network in the project the started instance will be attached to # A network in the project the started instance will be attached to
project_network = 'CloudComp' + str(group_number) + '-net' PROJECT_NETWORK = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance # The image to look for and use for the started instance
#ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image" # ubuntu_image_name = "Ubuntu 18.04 - Bionic Beaver - 64-bit - Cloud Based Image"
ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image" UBUNTU_IMAGE_NAME = "auto-sync/ubuntu-jammy-22.04-amd64-server-20240319-disk1.img"
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key # The public key to be used for SSH connection, please make sure, that you have the
# corresponding private key
# #
# id_rsa.pub should look like this (standard sshd pubkey format): # id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
#keypair_name = 'CloudComp30-keypair' KEYPAIR_NAME = 'srieger-pub'
keypair_name = "srieger-pub" PUB_KEY_FILE = '~/.ssh/id_rsa.pub'
pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 'm1.small' FLAVOR_NAME = 'm1.small'
# default region # default region
region_name = 'RegionOne' REGION_NAME = 'RegionOne'
# domain to use, "default" for local accounts, "hsfulda" for RZ LDAP, e.g., using fdaiXXXX as auth_username # domain to use, "default" for local accounts, formerly "hsfulda" for LDAP accounts etc.
# domain_name = "default" # domain_name = "default"
def main(): def main(): # noqa: C901 pylint: disable=too-many-branches,too-many-statements,too-many-locals,missing-function-docstring
########################################################################### ###########################################################################
# #
# get credentials # get credentials
@@ -61,12 +89,12 @@ def main():
########################################################################### ###########################################################################
provider = get_driver(Provider.OPENSTACK) provider = get_driver(Provider.OPENSTACK)
conn = provider(auth_username, conn = provider(AUTH_USERNAME,
auth_password, auth_password,
ex_force_auth_url=auth_url, ex_force_auth_url=AUTH_URL,
ex_force_auth_version='3.x_password', ex_force_auth_version='3.x_password',
ex_tenant_name=project_name, ex_tenant_name=PROJECT_NAME,
ex_force_service_region=region_name) ex_force_service_region=REGION_NAME)
# ex_domain_name=domain_name) # ex_domain_name=domain_name)
########################################################################### ###########################################################################
@@ -77,19 +105,19 @@ def main():
images = conn.list_images() images = conn.list_images()
image = '' image = ''
for img in images: for img in images:
if img.name == ubuntu_image_name: if img.name == UBUNTU_IMAGE_NAME:
image = img image = img
flavors = conn.list_sizes() flavors = conn.list_sizes()
flavor = '' flavor = ''
for flav in flavors: for flav in flavors:
if flav.name == flavor_name: if flav.name == FLAVOR_NAME:
flavor = conn.ex_get_size(flav.id) flavor = conn.ex_get_size(flav.id)
networks = conn.ex_list_networks() networks = conn.ex_list_networks()
network = '' network = ''
for net in networks: for net in networks:
if net.name == project_network: if net.name == PROJECT_NETWORK:
network = net network = net
########################################################################### ###########################################################################
@@ -101,14 +129,14 @@ def main():
print('Checking for existing SSH key pair...') print('Checking for existing SSH key pair...')
keypair_exists = False keypair_exists = False
for keypair in conn.list_key_pairs(): for keypair in conn.list_key_pairs():
if keypair.name == keypair_name: if keypair.name == KEYPAIR_NAME:
keypair_exists = True keypair_exists = True
if keypair_exists: if keypair_exists:
print(('Keypair ' + keypair_name + ' already exists. Skipping import.')) print('Keypair ' + KEYPAIR_NAME + ' already exists. Skipping import.')
else: else:
print('adding keypair...') print('adding keypair...')
conn.import_key_pair_from_file(keypair_name, pub_key_file) conn.import_key_pair_from_file(KEYPAIR_NAME, PUB_KEY_FILE)
for keypair in conn.list_key_pairs(): for keypair in conn.list_key_pairs():
print(keypair) print(keypair)
@@ -129,10 +157,12 @@ def main():
security_group_exists = True security_group_exists = True
if security_group_exists: if security_group_exists:
print(('Security Group ' + all_in_one_security_group.name + ' already exists. Skipping creation.')) print('Security Group ' + all_in_one_security_group.name + ' already exists. '
'Skipping creation.')
else: else:
all_in_one_security_group = conn.ex_create_security_group(security_group_name, all_in_one_security_group = conn.ex_create_security_group(security_group_name,
'network access for all-in-one application.') 'network access for '
'all-in-one application.')
conn.ex_create_security_group_rule(all_in_one_security_group, 'TCP', 80, 80) conn.ex_create_security_group_rule(all_in_one_security_group, 'TCP', 80, 80)
conn.ex_create_security_group_rule(all_in_one_security_group, 'TCP', 22, 22) conn.ex_create_security_group_rule(all_in_one_security_group, 'TCP', 22, 22)
@@ -145,14 +175,14 @@ def main():
# #
########################################################################### ###########################################################################
#hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh' hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh' # noqa: E501 pylint: disable=line-too-long
# testing / faafo dev branch: # testing / faafo dev branch:
hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/branch/dev_faafo/faafo/contrib/install.sh' # hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/branch/dev_faafo/faafo/contrib/install.sh' # noqa: E501 pylint: disable=line-too-long
userdata = '''#!/usr/bin/env bash userdata = '#!/usr/bin/env bash\n' \
curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \ f'curl -L -s {hsfd_faafo_cloud_init_script} | bash -s -- ' \
-i faafo -i messaging -r api -r worker -r demo '-i faafo -i messaging -r api -r worker -r demo\n'
''' print('\nUsing cloud-init userdata:\n"' + userdata + '"\n')
print('Checking for existing instance...') print('Checking for existing instance...')
instance_name = 'all-in-one' instance_name = 'all-in-one'
@@ -164,7 +194,7 @@ def main():
instance_exists = True instance_exists = True
if instance_exists: if instance_exists:
print(('Instance ' + testing_instance.name + ' already exists. Skipping creation.')) print('Instance ' + testing_instance.name + ' already exists. Skipping creation.')
exit() exit()
else: else:
print('Starting new all-in-one instance and wait until it is running...') print('Starting new all-in-one instance and wait until it is running...')
@@ -172,7 +202,7 @@ def main():
image=image, image=image,
size=flavor, size=flavor,
networks=[network], networks=[network],
ex_keyname=keypair_name, ex_keyname=KEYPAIR_NAME,
ex_userdata=userdata, ex_userdata=userdata,
ex_security_groups=[all_in_one_security_group]) ex_security_groups=[all_in_one_security_group])
conn.wait_until_running(nodes=[testing_instance], timeout=120, ssh_interface='private_ips') conn.wait_until_running(nodes=[testing_instance], timeout=120, ssh_interface='private_ips')
@@ -186,12 +216,12 @@ def main():
private_ip = None private_ip = None
if len(testing_instance.private_ips): if len(testing_instance.private_ips):
private_ip = testing_instance.private_ips[0] private_ip = testing_instance.private_ips[0]
print(('Private IP found: {}'.format(private_ip))) print(f'Private IP found: {private_ip}')
public_ip = None public_ip = None
if len(testing_instance.public_ips): if len(testing_instance.public_ips):
public_ip = testing_instance.public_ips[0] public_ip = testing_instance.public_ips[0]
print(('Public IP found: {}'.format(public_ip))) print(f'Public IP found: {public_ip}')
print('Checking for unused Floating IP...') print('Checking for unused Floating IP...')
unused_floating_ip = None unused_floating_ip = None
@@ -202,11 +232,11 @@ def main():
if not unused_floating_ip and len(conn.ex_list_floating_ip_pools()): if not unused_floating_ip and len(conn.ex_list_floating_ip_pools()):
pool = conn.ex_list_floating_ip_pools()[0] pool = conn.ex_list_floating_ip_pools()[0]
print(('Allocating new Floating IP from pool: {}'.format(pool))) print(f'Allocating new Floating IP from pool: {pool}')
unused_floating_ip = pool.create_floating_ip() unused_floating_ip = pool.create_floating_ip()
if public_ip: if public_ip:
print(('Instance ' + testing_instance.name + ' already has a public ip. Skipping attachment.')) print('Instance ' + testing_instance.name + ' already has a public ip. Skipping attachment.')
elif unused_floating_ip: elif unused_floating_ip:
conn.ex_attach_floating_ip_to_node(testing_instance, unused_floating_ip) conn.ex_attach_floating_ip_to_node(testing_instance, unused_floating_ip)
@@ -218,17 +248,20 @@ def main():
elif private_ip: elif private_ip:
actual_ip_address = private_ip actual_ip_address = private_ip
print('\n') print('\n\n#### Deployment finished\n\n')
print(('The Fractals app will be deployed to http://{}\n'.format(actual_ip_address))) print('After some minutes, as soon as cloud-init installed required packages and the\n'
'faafo app, (First App Application For OpenStack) fractals demo will be available\n'
f'at http://{actual_ip_address}\n')
print('You can use ssh to login to the instance using your private key. Default user name for official Ubuntu\n' print('You can use ssh to login to the instance using your private key. Default user name for official Ubuntu\n'
'Cloud Images is: ubuntu, so you can use, e.g.: "ssh -i ~/.ssh/id_rsa ubuntu@<floating-ip>" if your private\n' f'Cloud Images is: ubuntu, so you can use, e.g.: "ssh -i ~/.ssh/id_rsa ubuntu@{actual_ip_address}" if your\n'
'key is in the default location.\n\n' 'private key is in the default location.\n\n'
'After login, you can list or "ssh ubuntu@<floating-ip>" available fractals using "faafo list". To request\n' 'After login, you can list available fractals using "faafo list". \n'
'the generation of new fractals, you can use "faafo create".\n\n' 'To request the generation of new fractals, you can use "faafo create".\n\n'
'You can also see other options to use the faafo example cloud service using "faafo -h".\n\n' 'You can also see other options to use the faafo example cloud service using "faafo -h".\n\n'
'If you cannot start faafo command and/or do not see the webpage, you can check the Instance Console Log of\n' 'If you cannot start faafo command and/or do not see the webpage, you can check the Instance Console Log of\n'
'the instance, e.g., in OpenStack web interface.') 'the instance, e.g., in OpenStack web interface or look at "tail -f /var/log/cloud-init*.log" for the\n'
'cloud-init log files.\n')
if __name__ == '__main__': if __name__ == '__main__':

150
demo3-microservice.py → demo3_microservice.py
View File

@@ -1,51 +1,73 @@
"""Example for Cloud Computing Course Master AI / GSD"""
# This script demonstrates how to use libcloud to start an instance in an OpenStack environment.
# The script will create start multiple instances splitting up the faafo monolithic application into
# a (minimalistic but already scalable) microservice architecture.
# Also introduces the concept of different security groups and corresponding frontend/backend
# separation.
# Needed if the password should be prompted for:
# import getpass # import getpass
# import os import os
import sys
from libcloud.compute.providers import get_driver from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider from libcloud.compute.types import Provider
# reqs: # For our new Charmed OpenStack private cloud, we need to specify the path to the root
# services: nova, glance, neutron # CA certificate
# resources: 2 instances, 2 floating ips (1 keypair, 2 security groups) import libcloud.security
libcloud.security.CA_CERTS_PATH = ['./root-ca.crt']
# Disable SSL certificate verification (not recommended for production)
# libcloud.security.VERIFY_SSL_CERT = False
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username, # Please use 1-29 as environment variable GROUP_NUMBER to specify your group number.
# project etc., as coordinated in the lab sessions) # (will be used for the username, project etc., as coordinated in the lab sessions)
group_number = X group_number = os.environ.get('GROUP_NUMBER')
if group_number is None:
sys.exit('Please set the GROUP_NUMBER environment variable to your group number,\n'
'e.g., on Windows:\n'
' "$env:GROUP_NUMBER=0" or "set GROUP_NUMBER=0"\n'
'or on Linux/MacOS:\n'
' "export GROUP_NUMBER=0" or "set GROUP_NUMBER=0"')
# web service endpoint of the private cloud infrastructure # web service endpoint of the private cloud infrastructure
auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000' # auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000'
AUTH_URL = 'https://10.32.4.182:5000'
# auth_url = 'https://private-cloud2.informatik.hs-fulda.de:5000'
# your username in OpenStack # your username in OpenStack
auth_username = 'CloudComp' + str(group_number) AUTH_USERNAME = 'CloudComp' + str(group_number)
print(f'Using username: {AUTH_USERNAME}\n')
# your project in OpenStack # your project in OpenStack
project_name = 'CloudComp' + str(group_number) PROJECT_NAME = 'CloudComp' + str(group_number)
# A network in the project the started instance will be attached to # A network in the project the started instance will be attached to
project_network = 'CloudComp' + str(group_number) + '-net' PROJECT_NETWORK = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance # The image to look for and use for the started instance
#ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image" # ubuntu_image_name = "Ubuntu 18.04 - Bionic Beaver - 64-bit - Cloud Based Image"
ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image" UBUNTU_IMAGE_NAME = "auto-sync/ubuntu-jammy-22.04-amd64-server-20240319-disk1.img"
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key # The public key to be used for SSH connection, please make sure, that you have the
# corresponding private key
# #
# id_rsa.pub should look like this (standard sshd pubkey format): # id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
#keypair_name = 'CloudComp30-keypair' KEYPAIR_NAME = 'srieger-pub'
keypair_name = "srieger-pub" PUB_KEY_FILE = '~/.ssh/id_rsa.pub'
pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 'm1.small' FLAVOR_NAME = 'm1.small'
# default region # default region
region_name = 'RegionOne' REGION_NAME = 'RegionOne'
# domain to use, "default" for local accounts, "hsfulda" for RZ LDAP, e.g., using fdaiXXXX as auth_username # domain to use, "default" for local accounts, formerly "hsfulda" for LDAP accounts etc.
# domain_name = "default" # domain_name = "default"
def main(): def main(): # noqa: C901 pylint: disable=too-many-branches,too-many-statements,too-many-locals,missing-function-docstring
########################################################################### ###########################################################################
# #
# get credentials # get credentials
@@ -65,12 +87,12 @@ def main():
########################################################################### ###########################################################################
provider = get_driver(Provider.OPENSTACK) provider = get_driver(Provider.OPENSTACK)
conn = provider(auth_username, conn = provider(AUTH_USERNAME,
auth_password, auth_password,
ex_force_auth_url=auth_url, ex_force_auth_url=AUTH_URL,
ex_force_auth_version='3.x_password', ex_force_auth_version='3.x_password',
ex_tenant_name=project_name, ex_tenant_name=PROJECT_NAME,
ex_force_service_region=region_name) ex_force_service_region=REGION_NAME)
# ex_domain_name=domain_name) # ex_domain_name=domain_name)
########################################################################### ###########################################################################
@@ -82,19 +104,19 @@ def main():
images = conn.list_images() images = conn.list_images()
image = '' image = ''
for img in images: for img in images:
if img.name == ubuntu_image_name: if img.name == UBUNTU_IMAGE_NAME:
image = img image = img
flavors = conn.list_sizes() flavors = conn.list_sizes()
flavor = '' flavor = ''
for flav in flavors: for flav in flavors:
if flav.name == flavor_name: if flav.name == FLAVOR_NAME:
flavor = conn.ex_get_size(flav.id) flavor = conn.ex_get_size(flav.id)
networks = conn.ex_list_networks() networks = conn.ex_list_networks()
network = '' network = ''
for net in networks: for net in networks:
if net.name == project_network: if net.name == PROJECT_NETWORK:
network = net network = net
########################################################################### ###########################################################################
@@ -106,14 +128,14 @@ def main():
print('Checking for existing SSH key pair...') print('Checking for existing SSH key pair...')
keypair_exists = False keypair_exists = False
for keypair in conn.list_key_pairs(): for keypair in conn.list_key_pairs():
if keypair.name == keypair_name: if keypair.name == KEYPAIR_NAME:
keypair_exists = True keypair_exists = True
if keypair_exists: if keypair_exists:
print(('Keypair ' + keypair_name + ' already exists. Skipping import.')) print('Keypair ' + KEYPAIR_NAME + ' already exists. Skipping import.')
else: else:
print('adding keypair...') print('adding keypair...')
conn.import_key_pair_from_file(keypair_name, pub_key_file) conn.import_key_pair_from_file(KEYPAIR_NAME, PUB_KEY_FILE)
for keypair in conn.list_key_pairs(): for keypair in conn.list_key_pairs():
print(keypair) print(keypair)
@@ -134,9 +156,11 @@ def main():
security_group_exists = True security_group_exists = True
if security_group_exists: if security_group_exists:
print(('Worker Security Group ' + worker_security_group.name + ' already exists. Skipping creation.')) print('Worker Security Group ' + worker_security_group.name + ' already exists. '
'Skipping creation.')
else: else:
worker_security_group = conn.ex_create_security_group('worker', 'for services that run on a worker node') worker_security_group = conn.ex_create_security_group('worker', 'for services '
'that run on a worker node')
conn.ex_create_security_group_rule(worker_security_group, 'TCP', 22, 22) conn.ex_create_security_group_rule(worker_security_group, 'TCP', 22, 22)
print('Checking for existing controller security group...') print('Checking for existing controller security group...')
@@ -149,9 +173,11 @@ def main():
security_group_exists = True security_group_exists = True
if security_group_exists: if security_group_exists:
print(('Controller Security Group ' + controller_security_group.name + ' already exists. Skipping creation.')) print('Controller Security Group ' + controller_security_group.name + ' already exists. '
'Skipping creation.')
else: else:
controller_security_group = conn.ex_create_security_group('control', 'for services that run on a control node') controller_security_group = conn.ex_create_security_group('control', 'for services that '
'run on a control node')
conn.ex_create_security_group_rule(controller_security_group, 'TCP', 22, 22) conn.ex_create_security_group_rule(controller_security_group, 'TCP', 22, 22)
conn.ex_create_security_group_rule(controller_security_group, 'TCP', 80, 80) conn.ex_create_security_group_rule(controller_security_group, 'TCP', 80, 80)
conn.ex_create_security_group_rule(controller_security_group, 'TCP', 5672, 5672, conn.ex_create_security_group_rule(controller_security_group, 'TCP', 5672, 5672,
@@ -166,24 +192,18 @@ def main():
# #
########################################################################### ###########################################################################
#hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh' userdata = '#!/usr/bin/env bash\n' \
# testing / faafo dev branch: 'curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-' \
hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/branch/dev_faafo/faafo/contrib/install.sh' 'examples/raw/master/faafo/contrib/install.sh | bash -s -- ' \
'-i messaging -i faafo -r api\n'
userdata = '''#!/usr/bin/env bash print('\nUsing cloud-init userdata for controller:\n"' + userdata + '"\n')
curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i messaging -i faafo -r api
rabbitmqctl add_user faafo guest
rabbitmqctl set_user_tags faafo administrator
rabbitmqctl set_permissions -p / faafo ".*" ".*" ".*"
'''
print('Starting new app-controller instance and wait until it is running...') print('Starting new app-controller instance and wait until it is running...')
instance_controller_1 = conn.create_node(name='app-controller', instance_controller_1 = conn.create_node(name='app-controller',
image=image, image=image,
size=flavor, size=flavor,
networks=[network], networks=[network],
ex_keyname=keypair_name, ex_keyname=KEYPAIR_NAME,
ex_userdata=userdata, ex_userdata=userdata,
ex_security_groups=[controller_security_group]) ex_security_groups=[controller_security_group])
@@ -204,11 +224,12 @@ def main():
if not unused_floating_ip: if not unused_floating_ip:
pool = conn.ex_list_floating_ip_pools()[0] pool = conn.ex_list_floating_ip_pools()[0]
print(('Allocating new Floating IP from pool: {}'.format(pool))) print(f'Allocating new Floating IP from pool: {pool}')
unused_floating_ip = pool.create_floating_ip() unused_floating_ip = pool.create_floating_ip()
conn.ex_attach_floating_ip_to_node(instance_controller_1, unused_floating_ip) conn.ex_attach_floating_ip_to_node(instance_controller_1, unused_floating_ip)
print(('Controller Application will be deployed to http://%s' % unused_floating_ip.ip_address)) print(f'Controller Application will be deployed to http://{unused_floating_ip.ip_address}')
actual_ip_address = unused_floating_ip.ip_address
########################################################################### ###########################################################################
# #
@@ -218,6 +239,7 @@ def main():
# instance should not have a public ip? floating ips are assigned later # instance should not have a public ip? floating ips are assigned later
instance_controller_1 = conn.ex_get_node_details(instance_controller_1.id) instance_controller_1 = conn.ex_get_node_details(instance_controller_1.id)
ip_controller = ''
if instance_controller_1.public_ips: if instance_controller_1.public_ips:
ip_controller = instance_controller_1.public_ips[0] ip_controller = instance_controller_1.public_ips[0]
else: else:
@@ -229,17 +251,19 @@ def main():
# #
########################################################################### ###########################################################################
userdata = '''#!/usr/bin/env bash userdata = '#!/usr/bin/env bash\n' \
curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \ 'curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-' \
-i faafo -r worker -e 'http://%(ip_controller)s' -m 'amqp://faafo:guest@%(ip_controller)s:5672/' 'examples/raw/master/faafo/contrib/install.sh | bash -s -- ' \
''' % {'ip_controller': ip_controller} f'-i faafo -r worker -e "http://{ip_controller}" -m "amqp://faafo:guest@' \
f'{ip_controller}:5672/"\n'
print('\nUsing cloud-init userdata for worker:\n"' + userdata + '"\n')
print('Starting new app-worker-1 instance and wait until it is running...') print('Starting new app-worker-1 instance and wait until it is running...')
instance_worker_1 = conn.create_node(name='app-worker-1', instance_worker_1 = conn.create_node(name='app-worker-1',
image=image, image=image,
size=flavor, size=flavor,
networks=[network], networks=[network],
ex_keyname=keypair_name, ex_keyname=KEYPAIR_NAME,
ex_userdata=userdata, ex_userdata=userdata,
ex_security_groups=[worker_security_group]) ex_security_groups=[worker_security_group])
@@ -260,15 +284,23 @@ def main():
if not unused_floating_ip: if not unused_floating_ip:
pool = conn.ex_list_floating_ip_pools()[0] pool = conn.ex_list_floating_ip_pools()[0]
print(('Allocating new Floating IP from pool: {}'.format(pool))) print(f'Allocating new Floating IP from pool: {pool}')
unused_floating_ip = pool.create_floating_ip() unused_floating_ip = pool.create_floating_ip()
conn.ex_attach_floating_ip_to_node(instance_worker_1, unused_floating_ip) conn.ex_attach_floating_ip_to_node(instance_worker_1, unused_floating_ip)
print(('The worker will be available for SSH at %s' % unused_floating_ip.ip_address)) print(f'The worker will be available for SSH at {unused_floating_ip.ip_address}')
print('You can use ssh to login to the controller using your private key. After login, you can list available ' print('\n\n#### Deployment finished\n\n')
'fractals using "faafo list". To request the generation of new fractals, you can use "faafo create". ' print('After some minutes, as soon as cloud-init installed required packages and the\n'
'You can also see other options to use the faafo example cloud service using "faafo -h".') 'faafo app, (First App Application For OpenStack) fractals demo will be available\n'
f'at http://{actual_ip_address}\n')
print('You can use ssh to login to the controller using your private key.\n'
f'E.g., "ssh -i ~/.ssh/id_rsa ubuntu@{actual_ip_address}". After login,\n'
'you can list available fractals using "faafo list". To request the generation of\n'
'new fractals, you can use "faafo create". \n'
'You can also see other options to use the faafo example cloud service using '
'"faafo -h".')
if __name__ == '__main__': if __name__ == '__main__':

97
destroy-all-demo-instances.py
View File

@@ -1,97 +0,0 @@
# import getpass
# import os
# import libcloud.security
import time
from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider
# reqs:
# services: nova, glance, neutron
# resources: 2 instances (m1.small), 2 floating ips (1 keypair, 2 security groups)
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions)
group_number = X
# web service endpoint of the private cloud infrastructure
auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000'
# your username in OpenStack
auth_username = 'CloudComp' + str(group_number)
# your project in OpenStack
project_name = 'CloudComp' + str(group_number)
# default region
region_name = 'RegionOne'
# domain to use, "default" for local accounts, "hsfulda" for RZ LDAP, e.g., using fdaiXXXX as auth_username
domain_name = "default"
def main():
###########################################################################
#
# get credentials
#
###########################################################################
# if "OS_PASSWORD" in os.environ:
# auth_password = os.environ["OS_PASSWORD"]
# else:
# auth_password = getpass.getpass("Enter your OpenStack password:")
auth_password = "demo"
###########################################################################
#
# create connection
#
###########################################################################
# libcloud.security.VERIFY_SSL_CERT = False
provider = get_driver(Provider.OPENSTACK)
conn = provider(auth_username,
auth_password,
ex_force_auth_url=auth_url,
ex_force_auth_version='3.x_password',
ex_tenant_name=project_name,
ex_force_service_region=region_name,
ex_domain_name=domain_name)
###########################################################################
#
# clean up resources from previous demos
#
###########################################################################
# destroy running demo instances
for instance in conn.list_nodes():
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller',
'app-services', 'app-api-1', 'app-api-2']:
print(('Destroying Instance: %s' % instance.name))
conn.destroy_node(instance)
# wait until all nodes are destroyed to be able to remove depended security groups
nodes_still_running = True
while nodes_still_running:
nodes_still_running = False
time.sleep(3)
instances = conn.list_nodes()
for instance in instances:
# if we see any demo instances still running continue to wait for them to stop
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller',
'app-services', 'app-api-1', 'app-api-2']:
nodes_still_running = True
print('There are still instances running, waiting for them to be destroyed...')
# delete security groups
for group in conn.ex_list_security_groups():
if group.name in ['control', 'worker', 'api', 'services']:
print(('Deleting security group: %s' % group.name))
conn.ex_delete_security_group(group)
if __name__ == '__main__':
main()

126
destroy_all_demo_instances.py Normal file
View File

@@ -0,0 +1,126 @@
"""Example for Cloud Computing Course Master AI / GSD"""
# This script demonstrates how to use libcloud to cleanup all instances used in the demos
# for our OpenStack private cloud environment.
# import getpass
import os
import sys
import time
from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider
# For our new Charmed OpenStack private cloud, we need to specify the path to the
# root CA certificate
import libcloud.security
libcloud.security.CA_CERTS_PATH = ['./root-ca.crt']
# Disable SSL certificate verification (not recommended for production)
# libcloud.security.VERIFY_SSL_CERT = False
# Please use 1-29 for X in the following variable to specify your group number.
# (will be used for the username, project etc., as coordinated in the lab sessions)
group_number = os.environ.get('GROUP_NUMBER')
if group_number is None:
sys.exit('Please set the GROUP_NUMBER environment variable to your group number,\n'
'e.g., on Windows:\n'
' "$env:GROUP_NUMBER=0" or "set GROUP_NUMBER=0"\n'
'or on Linux/MacOS:\n'
' "export GROUP_NUMBER=0" or "set GROUP_NUMBER=0"')
###############################################################################################
#
# no changes necessary below this line in this example
#
###############################################################################################
# web service endpoint of the private cloud infrastructure
# auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000'
AUTH_URL = 'https://10.32.4.182:5000'
# auth_url = 'https://private-cloud2.informatik.hs-fulda.de:5000'
# your username in OpenStack
AUTH_USERNAME = 'CloudComp' + str(group_number)
# your project in OpenStack
PROJECT_NAME = 'CloudComp' + str(group_number)
# A network in the project the started instance will be attached to
PROJET_NETWORK = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance
# ubuntu_image_name = "Ubuntu 18.04 - Bionic Beaver - 64-bit - Cloud Based Image"
UBUNTU_IMAGE_NAME = "auto-sync/ubuntu-jammy-22.04-amd64-server-20240319-disk1.img"
# default region
REGION_NAME = 'RegionOne'
# domain to use, "default" for local accounts, formerly "hsfulda" for LDAP accounts etc.
# domain_name = "default"
def main(): # noqa: C901 pylint: disable=too-many-branches,too-many-statements,too-many-locals,missing-function-docstring
###########################################################################
#
# get credentials
#
###########################################################################
# if "OS_PASSWORD" in os.environ:
# auth_password = os.environ["OS_PASSWORD"]
# else:
# auth_password = getpass.getpass("Enter your OpenStack password:")
auth_password = "demo"
###########################################################################
#
# create connection
#
###########################################################################
# libcloud.security.VERIFY_SSL_CERT = False
provider = get_driver(Provider.OPENSTACK)
conn = provider(AUTH_USERNAME,
auth_password,
ex_force_auth_url=AUTH_URL,
ex_force_auth_version='3.x_password',
ex_tenant_name=PROJECT_NAME,
ex_force_service_region=REGION_NAME)
# ex_domain_name=domain_name)
###########################################################################
#
# clean up resources from previous demos
#
###########################################################################
# destroy running demo instances
for instance in conn.list_nodes():
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3',
'app-controller',
'app-services', 'app-api-1', 'app-api-2']:
print(f'Destroying Instance: {instance.name}')
conn.destroy_node(instance)
# wait until all nodes are destroyed to be able to remove depended security groups
nodes_still_running = True
while nodes_still_running:
nodes_still_running = False
time.sleep(3)
instances = conn.list_nodes()
for instance in instances:
# if we see any demo instances still running continue to wait for them to stop
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3',
'app-controller', 'app-services', 'app-api-1', 'app-api-2']:
nodes_still_running = True
print('There are still instances running, waiting for them to be destroyed...')
# delete security groups
for group in conn.ex_list_security_groups():
if group.name in ['control', 'worker', 'api', 'services']:
print(f'Deleting security group: {group.name}')
conn.ex_delete_security_group(group)
if __name__ == '__main__':
main()

21
root-ca.crt Normal file
View File

@@ -0,0 +1,21 @@
-----BEGIN CERTIFICATE-----
MIIDazCCAlOgAwIBAgIUEQjSqiZ86fhawQU09G8hn3i9dIwwDQYJKoZIhvcNAQEL
BQAwPTE7MDkGA1UEAxMyVmF1bHQgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkg
KGNoYXJtLXBraS1sb2NhbCkwHhcNMjQwMzI1MTMxNDU0WhcNMzQwMzIzMTIxNTIz
WjA9MTswOQYDVQQDEzJWYXVsdCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAo
Y2hhcm0tcGtpLWxvY2FsKTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
ALcax5l2zU1ELCtF/k6yq3HUv7gFq6f/sl8rX0VRhzZyEy4hcMgRSZDAM14viTVZ
8d7+ptY3+GwuLSpEY2UUlX5kJSDb4pUNRXDhxzRatbByG8pr5FQE8pX9W7y4C0TU
3PQA4uIjAsPFKayFxXjJjOQN0HX3K6MCQz/BTV81U3fmdFrKma3x/PXyUYndjQH6
zlIiQSdYh7FMTbS2FlpvwWbT9zKOpp+2M0odI8Y8fjCSUdSdKDFhVu02zQTq6/h0
Q1/sNHz2IP9F83sNW+ro0bvv5CJ2iCyAk/RiFoB+RoSO6HncOtYHxa/guwTy4eHh
VQVJXkEI2PutCw6S3lWqLEcCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
EwEB/wQFMAMBAf8wHQYDVR0OBBYEFDKQHfpVHJgge6RRC5uDwMByLbV3MB8GA1Ud
IwQYMBaAFDKQHfpVHJgge6RRC5uDwMByLbV3MA0GCSqGSIb3DQEBCwUAA4IBAQCa
ajIRi/+7Yy7l46yFURLyELMWayRpdx2KCxIuAiSkTlNSVOPCmwZvgnYLPVffXWpt
IXJGQk//9+5q18LiZat5MbvUU3ffLc/ZCxIeQiWNuYKziLYNFHmpfMvxNxzAJ6Pi
2fj5ZP/cA4Vie3M1iHfdSXmYPvyw76i9/sA2+F7Wy8fzK53S1OaMaeADNGljHTaW
ovRxreLKJZybqe/YWlcEiP4dC4VMHLl+H5RmZ5ojrRiy1c3uUssNnIJU+ilkY8TP
0VV17+wQBaJbbp4jh8acwvOJbN8Y1EHQWhxkEf3PfjJRv+b1NI/Iai27DfYto7Dm
rZvaFnAMCcyFXyJv3WdJ
-----END CERTIFICATE-----