[PYTHON] Extract the Azure service list

I made it because I needed to make a list in my work. ~~ I don't like the culture of making a list of anything in Excel. ~~

For the scraping method, I referred to AWS service list (2019/03 version) * Updated on 2020/09/05.

Azure service list

AI + machine learning

DevOps

ID

Web

Windows Virtual Desktop

container

Computing

storage

Security

Database

network

Hybrid environment

Blockchain

media

Internet of Things (IoT)

mobile

Migration

Developer tools

Management and governance

Integration

Mixed reality

analysis

Code for scraping (python)

import requests
from bs4 import BeautifulSoup
import re
import copy

def parse_html():
    url = 'https://azure.microsoft.com/ja-jp/services/'
    r = requests.get(url)
    soup = BeautifulSoup(r.content, 'lxml')

    groups = []
    service = []

    current_category = ''

    for div1 in soup.select('div#products-list'):
        for div2 in div1.select('div.row'):
            category = div2.select('h2.product-category')
            if not category:
                for div3 in div2.select('div.column.medium-6'):
                    s = {}
                    span = div3.select('span', limit=1)
                    href = div3.select('a', limit=1)
                    description = div3.select('p.text-body4', limit=1)

                    if len(span) > 0 and len(href) > 0 and len(description) > 0:
                        s['service_name'] = span[0].text.rstrip('\n')
                        s['href'] = href[0].get('href').rstrip('\n')
                        s['description'] = description[0].text.rstrip('\n')
                        service.append(s)

            else:
                if current_category != category[0].text:
                    if current_category == '':
                        current_category = category[0].text
                    elif len(service) > 0:
                        group = {}
                        tmp = copy.deepcopy(service)
                        group['category_name'] = current_category
                        group['service'] = tmp

                        groups.append(group)

                        current_category = category[0].text
                        service.clear()

        if len(service) > 0:
            group = {}
            group['category_name'] = current_category
            group['service'] = service
            groups.append(group)

    return (groups)


def print_markdown(services):
    base_url = 'https://azure.microsoft.com'

    for s in services:
        print('### %s' % (s['category_name']))

        print()
        print('|Service name|Explanation|')
        print('| --- | --- |')

        for service in s['service']:
            name = service['service_name']
            description = service['description']
            if (service['href'].startswith('/')):
                href = base_url + service['href']
            else:
                href = service['href']

            print('| [%s](%s)|%s|'%(name,href,description))
        print()


print_markdown(parse_html())

Reference site

https://qiita.com/moritalous/items/31a56acbf2ce367b712d