Машины для очистки веб-страниц

Есть ли способ уменьшить количество строк в этом коде и сделать его лучше с точки зрения Python? Программа собирает данные из truecar.com а затем помещает их в мою базу данных.

Любая помощь будет высоко оценен.

Код

import mysql.connector
import requests
import math
 

# takes the car's name
requested_car_name = input('Enter car name: ')

# injects the car's name into the URL

url="https://www.truecar.com/abp/api/vehicles/used/listings"
payload = {
    'city': 'holtsville',
    'collapse': 'true',
    'fallback': 'true',
    'include_incentives': 'true',
    'include_targeted_incentives': 'true',
    'make_slug': requested_car_name,
    'new_or_used': 'u',
    'per_page': '30',
    'postal_code': '',
    'search_event': 'true',
    'sort[]': 'best_match',
    'sponsored': 'true',
    'state': 'ny',
    'page': '1'}


jsonData = requests.get(url, params=payload).json()
total = jsonData['total']
total_pages = math.ceil(total/30)

total_pages_input = input(
    'There are %s pages to iterate.nEnter the number of pages to go through or type ALL: ' % total_pages)
if total_pages_input.upper() == 'ALL':
    total_pages = total_pages
else:
    total_pages = int(total_pages_input)

values = []
for page in range(1, total_pages+1):
    if page == 1:
        car_listings = jsonData['listings']
    else:
        payload.update({'page': '%s' % page})
        jsonData = requests.get(url, params=payload).json()
        car_listings = jsonData['listings']

    for listing in car_listings:
        vehicle = listing['vehicle']
        ex_color = vehicle['exterior_color']
        in_color = vehicle['interior_color']
        location = vehicle['location']
        price = vehicle['list_price']
        make = vehicle['make']
        model = vehicle['model']
        mileage = vehicle['mileage']
        style = vehicle['style']
        year = vehicle['year']
        engine = vehicle['engine']
        accidentCount = vehicle['condition_history']['accidentCount']
        ownerCount = vehicle['condition_history']['ownerCount']
        isCleanTitle = vehicle['condition_history']['titleInfo']['isCleanTitle']
        isFrameDamaged = vehicle['condition_history']['titleInfo']['isFrameDamaged']
        isLemon = vehicle['condition_history']['titleInfo']['isLemon']
        isSalvage = vehicle['condition_history']['titleInfo']['isSalvage']
        isTheftRecovered = vehicle['condition_history']['titleInfo']['isTheftRecovered']

        values.append((ex_color, in_color, location, price, make, model, mileage,
                       style, year, engine, accidentCount, ownerCount, isCleanTitle, isFrameDamaged,
                       isLemon, isSalvage, isTheftRecovered))
    print('Completed: Page %s of %s' % (page, total_pages))


# checks the connection
print('CONNECTING ...')

mydb = mysql.connector.connect(
    host="xxx",
    user="xxx",
    password="xxx",
    port="xxx",
    database="xxx"
)

print('CONNECTED')

# checking the connection is done

my_cursor = mydb.cursor(buffered=True)
# create_command = ''' create table car_information (exterior_color varchar(255), interior_color varchar(255),location varchar(255),price varchar(255),make varchar(255),model varchar(255),mileage varchar(255),
#         style varchar(255),year varchar(255),engine varchar(255),accidentCount varchar(255),ownerCount varchar(255),isCleanTitle varchar(255),isFrameDamaged varchar(255),
#         isLemon varchar(255), isSalvage varchar(255),isTheftRecovered varchar(255))'''

# my_cursor.execute(create_command)
# print('created')
insert_command = '''INSERT INTO car_information (exterior_color, interior_color,location,price,make,model,mileage,
        style,year,engine,accidentCount,ownerCount,isCleanTitle,isFrameDamaged,
        isLemon, isSalvage,isTheftRecovered) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'''
my_cursor.executemany(insert_command, values)
mydb.commit()

print(my_cursor.rowcount, "Record Inserted")

mydb.close()

• Функции записи; в настоящее время у вас нет ни одного
• requested_car_name имя обманчиво (вместе с Enter car name подсказка), потому что это фактически используется для автомобиля делать параметр
• Логика fetch / prompt / fetch для запроса количества страниц немного неудобна. Вместо этого просто попросите заранее указать ограничение на количество страниц.
• Избегайте if page == 1 / else блок и просто пролистайте страницы.
• Разделите методы HTTP-выборки, депагинации и формирования данных
• Используйте сеанс запросов для сохранения файлов cookie, применения заголовков и обеспечения повторного использования соединения.
• Не используйте кортеж для подготовленного оператора; использовать словарь и %(keyword)s синтаксис чтобы избежать ошибок несовпадения кортежей
• Опустите свой DDL (create table) из источника вашего приложения

Следующий предлагаемый источник работает успешно, но вообще не использует базу данных; эта часть зависит от вас.

from itertools import count
from typing import Any, Dict, Optional, Iterable, Tuple

import math

from requests import Session


BASE_PARAMS = {
    'city': 'denver',
    'collapse': 'true',
    'fallback': 'true',
    'include_incentives': 'true',
    'include_targeted_incentives': 'true',
    'new_or_used': 'u',
    'per_page': 30,
    'postal_code': '80210',
    'search_event': 'true',
    'sort[]': 'best_match',
    'sponsored': 'true',
    'state': 'ny',
}


def get_truecar(session: Session, make_name: str, page: int) -> Dict[str, Any]:
    with session.get(
        'https://www.truecar.com/abp/api/vehicles/used/listings',
        params={
            **BASE_PARAMS,
            'make_slug': make_name.lower(),
            'page': page,
        },
    ) as response:
        response.raise_for_status()
        return response.json()


def depaginate_truecar(
    session: Session, make_name: str, max_pages: Optional[int],
) -> Iterable[Tuple[Dict[str, Any], int, int]]:
    if max_pages is None:
        pages = count(1)
    else:
        pages = range(1, max_pages + 1)

    for page in pages:
        doc = get_truecar(session, make_name, page=page)
        total = doc.get('total')
        per_page = int(doc['per_page'])

        if total is None:
            n_pages = 1
        else:
            n_pages = math.ceil(total / per_page)
            if max_pages is not None:
                n_pages = min(max_pages, n_pages)

        yield doc, page, n_pages

        if total is None or page >= n_pages:
            break


def reshape(doc: Dict[str, Any]) -> Iterable[Dict[str, Any]]:
    for listing in doc['listings']:
        vehicle = listing['vehicle']

        yield {
            'ex_color': vehicle['exterior_color'],
            'in_color': vehicle['interior_color'],
            'location': vehicle['location'],
            'price': vehicle['list_price'],
            'make': vehicle['make'],
            'model': vehicle['model'],
            'mileage': vehicle['mileage'],
            'style': vehicle['style'],
            'year': vehicle['year'],
            'engine': vehicle['engine'],
            'accidentCount': vehicle['condition_history']['accidentCount'],
            'ownerCount': vehicle['condition_history']['ownerCount'],
            'isCleanTitle': vehicle['condition_history']['titleInfo']['isCleanTitle'],
            'isFrameDamaged': vehicle['condition_history']['titleInfo']['isFrameDamaged'],
            'isLemon': vehicle['condition_history']['titleInfo']['isLemon'],
            'isSalvage': vehicle['condition_history']['titleInfo']['isSalvage'],
            'isTheftRecovered': vehicle['condition_history']['titleInfo']['isTheftRecovered'],
        }


def main():
    car_name = input('Enter car name: ')
    max_pages = input('Enter page limit, or press enter for none: ')
    if max_pages == '':
        max_pages = None
    else:
        max_pages = int(max_pages)

    cars = []

    with Session() as session:
        session.headers = {
            'User-Agent': 'Mozilla/5.0',
            'Accept': 'application/json',
            'Referer': 'https://www.truecar.com/shop/used/?filterType=brand',
        }

        for doc, page, n_pages in depaginate_truecar(session, car_name, max_pages):
            print(f'{page}/{n_pages}')
            cars.extend(reshape(doc))


if __name__ == '__main__':
    main()