{"componentChunkName":"component---src-templates-material-js","path":"/workshops/2021/11/26/parsing-s-pomoshchyu-python-urok-4/","result":{"pageContext":{"pageParameters":{"isPreview":false,"isMirror":false,"delay_to_show_donate_popup_seconds":7,"about_page":false,"games_page":true,"games_page_link_in_header":false,"donate_popup_show_ru":true,"donate_popup_title_ru":"Без вас — нас не будет","donate_popup_content_ru":"Помогите нам добывать и рассказывать важные истории. Участие каждого — важно!","donate_popup_show_en":true,"donate_popup_title_en":"We can’t do it alone!","donate_popup_content_en":"Help us uncover and share the stories that matter. Every contribution makes a difference","display_regular_donors_banner":false,"showPatreonMethod":true,"useNewMaterialDonationBanner":true,"new_donations_page_design":true,"donations_popup_variants":{"ru":[{"id":"kc19b","title":"Сохраните журналистику для людей в России ","subtitle":"Читают нас прежде всего люди в России, но будущее «Важных историй» зависит от тех, кто живет за пределами страны. Поддержите нас ежемесячным донатом — сохраните независимую журналистику для ваших близких, которые остаются в России.","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"},{"id":"33tc9","title":"У пропаганды есть миллиарды из бюджета. У нас — только вы","subtitle":"Перемены возможны, если независимые от власти журналисты продолжают свою работу день за днем, год за годом. Мы рассказываем важные истории уже шесть лет. Помогите нам продолжать.","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"}],"en":[{"id":"nj1py","title":"Important stories don’t tell themselves","subtitle":"Independent journalism comes at a cost. We answer to no owners or advertisers — only to you, our readers.  Your ongoing support allows us to keep exposing the truth. Without you, we won’t be here.","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"},{"id":"xdfc1","title":"Help us outlast the Kremlin censorship","subtitle":"Censorship lose to journalism that keeps showing up, year after year. Monthly donors are what make that possible","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"},{"id":"ck121","title":"State propaganda has billions to spend. We have only you to rely on","subtitle":"Change happens when independent journalists keep showing up — day after day, year after year. Six years in, we're still showing up. Help us keep going with a monthly donation.","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"}]},"feedback_page":true,"donorsCount":0,"showAuthorsRequestAfterMaterial":true,"radioPage":true,"radioExists":{"ru":true,"en":false},"singlePlaylistPath":{"ru":"/radio/2025/12/22/razgovori-ovazhnikh/"},"with_bookmarks":true,"news_feed":true,"news_feed_skeleton":false,"news_page_redesign":true,"with_topics":true,"materials_topics":false,"topics_page":true,"with_pdf_share":false,"with_native_share":false,"preferred_source_page":true,"charon_new_form":true,"preferred_source_promos":true,"charon_en_page":true},"pk":381,"header":"Парсинг с помощью Python. Урок 4","lead":"Учимся оптимизированно переходить по ссылкам и скрейпить сайт с помощью библиотеки Scrapy","category":"workshops","entry_image":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-1920x960px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-1920x960px_2x.webp","for_width":1920,"for_height":960,"real_width_1x":1920,"real_width_2x":3840,"real_height_1x":960,"real_height_2x":1920},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-1920x960px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-1920x960px_2x.jpg","for_width":1920,"for_height":960,"real_width_1x":1920,"real_width_2x":3840,"real_height_1x":960,"real_height_2x":1920},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-1272x636px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-1272x636px_2x.webp","for_width":1272,"for_height":636,"real_width_1x":1272,"real_width_2x":2544,"real_height_1x":636,"real_height_2x":1272},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-1272x636px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-1272x636px_2x.jpg","for_width":1272,"for_height":636,"real_width_1x":1272,"real_width_2x":2544,"real_height_1x":636,"real_height_2x":1272},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-912x456px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-912x456px_2x.webp","for_width":912,"for_height":456,"real_width_1x":912,"real_width_2x":1824,"real_height_1x":456,"real_height_2x":912},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-912x456px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-912x456px_2x.jpg","for_width":912,"for_height":456,"real_width_1x":912,"real_width_2x":1824,"real_height_1x":456,"real_height_2x":912},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-600x400px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-600x400px_2x.webp","for_width":600,"for_height":400,"real_width_1x":600,"real_width_2x":1200,"real_height_1x":400,"real_height_2x":800},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-600x400px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-600x400px_2x.jpg","for_width":600,"for_height":400,"real_width_1x":600,"real_width_2x":1200,"real_height_1x":400,"real_height_2x":800},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-360x360px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-360x360px_2x.webp","for_width":360,"for_height":360,"real_width_1x":360,"real_width_2x":720,"real_height_1x":360,"real_height_2x":720},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-360x360px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/e71540f098cc4fb8a53dc261b7d60e36-360x360px_2x.jpg","for_width":360,"for_height":360,"real_width_1x":360,"real_width_2x":720,"real_height_1x":360,"real_height_2x":720}],"entry_image_credit":"","blocks":[{"data":"Продолжаем изучать парсинг на Python. В прошлом уроке мы познакомились с библиотекой Scrapy и попробовали с ее помощью собрать информацию о книгах из одного раздела онлайн-магазина. А сегодня мы соберем информацию обо всех книгах. Сделаем это с помощью специального класса — crawlspider. Он позволяет легко и быстро искать ссылки на странице и переходить по ним. ","type":"p","className":""},{"data":{"id":3274,"data":{"url":"https://www.youtube.com/embed/Dqp7H49T13o","text":"Съемка и монтаж: Глеб Лиманский","caption":"","provider":"youtube","rawInput":"https://www.youtube.com/watch?v=Dqp7H49T13o","ratio_mobile":"16:9","ratio_desktop":"16:9"},"category":"video-embed"},"type":"video-embed"},{"data":"Откроем проект, с которым работали на прошлом уроке и создадим еще одного паука. Для этого переходим в папку spiders и создаем новый файл с расширением «py». Назовем его «all_books_spider.py».","type":"p","className":""},{"data":{"id":3257,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/e162b7d179284323b21ab124a981f74c-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/e162b7d179284323b21ab124a981f74c-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":746,"real_height_2x":1492},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/e162b7d179284323b21ab124a981f74c-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/e162b7d179284323b21ab124a981f74c-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":746,"real_height_2x":1492}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Импорт библиотек","type":"h2","className":""},{"data":"Как и в прошлый раз, мы начинаем наш проект с импорта библиотеки.","type":"p","className":""},{"data":"Пишем: import scrapy","type":"p","className":""},{"data":"Кроме этого, нам нужно импортировать библиотеки, которые будут собирать ссылки со страницы.&nbsp;","type":"p","className":""},{"data":"Пишем:","type":"p","className":""},{"data":"from scrapy.spiders import CrawlSpider, Rule","type":"p","className":""},{"data":"from scrapy.linkextractors import LinkExtractor","type":"p","className":""},{"data":{"id":3258,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/78215730f2524cd1a695638d1d594ea6-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/78215730f2524cd1a695638d1d594ea6-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":556,"real_height_2x":1112},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/78215730f2524cd1a695638d1d594ea6-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/78215730f2524cd1a695638d1d594ea6-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":556,"real_height_2x":1112}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Создаем класс","type":"h2","className":""},{"data":"Как и в прошлом уроке, создаем класс, в который запишем правила скрейпинга. Только на этот раз в скобках вместо scrapy.Spider мы напишем CrawlSpider:&nbsp;class BookSpider(CrawlSpider). ","type":"p","className":""},{"data":"Даем пауку имя: name = 'all_books'. Задаем начальную ссылку — это будет стартовая страница онлайн-магазина:&nbsp;start_urls = ['<a href=\"https://book24.ru/\" data-type=\"usual\" target=\"_blank\">https://book24.ru/</a>'].","type":"p","className":""},{"data":{"id":3259,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/524217cfd0d0491bb1155334ecb3d25d-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/524217cfd0d0491bb1155334ecb3d25d-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":617,"real_height_2x":1234},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/524217cfd0d0491bb1155334ecb3d25d-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/524217cfd0d0491bb1155334ecb3d25d-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":617,"real_height_2x":1234}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Сбор ссылок","type":"h2","className":""},{"data":"Теперь нужно написать правило сбора ссылок.&nbsp;","type":"p","className":""},{"data":"Пишем: rules =. Cоздаем кортеж. Кортеж похож на список, отличие в том, что список — изменяемый тип данных, а кортеж нет.&nbsp;","type":"p","className":""},{"data":{"id":3260,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/4afa5177e0254757b52295c3d4dce3b9-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/4afa5177e0254757b52295c3d4dce3b9-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":667,"real_height_2x":1334},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/4afa5177e0254757b52295c3d4dce3b9-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/4afa5177e0254757b52295c3d4dce3b9-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":667,"real_height_2x":1334}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Чтобы понять, какое правило задать, откроем сайт.&nbsp;Мы видим несколько категорий с книгами. Можно переходить по ним поэтапно, собирая информацию о каждой книжке на каждой странице каждого раздела. И так пройтись по всем разделам. Ссылку для каждого перехода нам бы пришлось задавать с помощью css-селектора, как мы это делали в прошлый раз. Класс Crawl spider позволяет выполнить задачу проще и быстрее.","type":"p","className":""},{"data":"Давайте посмотрим на ссылку одного из разделов. Например, художественная литература.&nbsp;","type":"p","className":""},{"data":{"id":3261,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/579c91c299df47b5bb0101383a1ded6a-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/579c91c299df47b5bb0101383a1ded6a-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":624,"real_height_2x":1247},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/579c91c299df47b5bb0101383a1ded6a-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/579c91c299df47b5bb0101383a1ded6a-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":624,"real_height_2x":1247}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Скопируем ссылку и посмотрим на нее. Чтобы эта строка не влияла на код, закомментируем ее, для этого нужно написать в начало решетку: #<a href=\"https://book24.ru/catalog/fiction-1592/\" data-type=\"usual\" target=\"_blank\">https://book24.ru/catalog/fiction-1592/</a>.","type":"p","className":""},{"data":"Мы видим, что после основного названия сайта (https://book24.ru) есть слово «catalog», а затем конкретный раздел каталога: fiction. Если мы зайдем в любой другой раздел, в ссылке тоже будет слово «catalog». Значит, нам нужно создать правило, чтобы паук проходился по всем ссылкам, в которых есть «catalog». Так мы соберем все разделы.&nbsp;","type":"p","className":""},{"data":"Теперь зайдем на страницу с книгой. Скопируем ссылку и посмотрим на нее. #https://book24.ru/product/portret-doriana-greya-1597309/.","type":"p","className":""},{"data":"После основного названия сайта, есть слово «product». Такое слово есть на странице с каждой книгой. Значит, чтобы собрать все книги в разделе, нужно пройтись по всем ссылкам со словом «product».","type":"p","className":""},{"data":"Запишем наше правило: Rule(LinkExtractor(allow='catalog')). Команда allow говорит, чтобы паук перешел по всем ссылкам, в которых есть слово «catalog». С помощью deny мы могли бы задать, по ссылкам с каким словом переходить не нужно.&nbsp;","type":"p","className":""},{"data":"Вторым правилом зададим переход по страницам книг, то есть ссылкам со словом «product»: Rule(LinkExtractor(allow='product').","type":"p","className":""},{"data":"За то, что паук делает на странице, отвечает callback. Ему мы передадим функцию parse_items, в которой будет лежать информация, что именно нужно собрать со страницы: Rule(LinkExtractor(allow='product'), callback='parse_items')","type":"p","className":""},{"data":"Мы получили два правила. Первое говорит пауку, что нужно пройти по всем ссылкам со словом «catalog», после этого включается второе правило, которое говорит переходить по ссылкам со словом&nbsp;«product» и собирать нужную информацию.&nbsp;","type":"p","className":""},{"data":{"id":3262,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/de5f75efba0e42f7935b9f02427786ff-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/de5f75efba0e42f7935b9f02427786ff-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":664,"real_height_2x":1330},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/de5f75efba0e42f7935b9f02427786ff-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/de5f75efba0e42f7935b9f02427786ff-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":664,"real_height_2x":1330}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Функция","type":"h2","className":""},{"data":"В прошлом уроке первую созданную функцию мы называли parse. Работая с классом CrawlSpider, нельзя использовать это название. Иначе код будет работать некорректно. Это важно помнить. Поэтому мы назвали нашу функцию parse_items. Как собирать информацию со страницы книги, мы подробно разбирали на прошлом уроке.&nbsp;","type":"p","className":""},{"data":"Напишем ключевое слово yield и создадим словарь. Собирать необходимые нам элементы будем с помощью css-селекторов.&nbsp;","type":"p","className":""},{"data":"Путь к названию и количеству покупок скопируем из предыдущего паука.","type":"p","className":""},{"data":"'name': response.css('h1.product-detail-page__title::text').get().strip(),","type":"p","className":""},{"data":"'buy': response.css('p.product-detail-page__purchased-text::text').get().split()[1],&nbsp;","type":"p","className":""},{"data":{"id":3272,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/44034c72db1842b3b28da9f73425bd59-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/44034c72db1842b3b28da9f73425bd59-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":220,"real_height_2x":442},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/44034c72db1842b3b28da9f73425bd59-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/44034c72db1842b3b28da9f73425bd59-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":220,"real_height_2x":442}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Чтобы полученный датасет был хорошо структурирован, соберем информацию обо всех разделах, которым принадлежит книга. Они как матрешка вкладываются друг в друга. Для примера перейдем в раздел <strong>«</strong>художественная литература<strong>»</strong> <strong>→ «</strong>проза<strong>»</strong> <strong>→ «</strong>историческая проза<strong>»</strong> и откроем страницу с книгой. ","type":"p","className":""},{"data":{"id":3263,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/be4ac3efa8ab46ed996148c829027392-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/be4ac3efa8ab46ed996148c829027392-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":560,"real_height_2x":1121},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/be4ac3efa8ab46ed996148c829027392-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/be4ac3efa8ab46ed996148c829027392-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":560,"real_height_2x":1121}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Scrapy Shell","type":"h2","className":""},{"data":"Чтобы собрать информацию об этих разделах, перейдем в scrapy shell. Для этого наберем в терминале команду scrapy shell.","type":"p","className":""},{"data":"После этого передадим команде fetch страницу книги: fetch('<a href=\"https://book24.ru/product/odin-den-v-drevnem-mire-zapiski-puteshestvennika-vo-vremeni-6024803/\" data-type=\"usual\" target=\"_blank\">https://book24.ru/product/odin-den-v-drevnem-mire-zapiski-puteshestvennika-vo-vremeni-6024803/</a>').","type":"p","className":""},{"data":"Мы увидели ответ 200, значит, с сайтом можно работать.&nbsp;","type":"p","className":""},{"data":{"id":3265,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/7c6d529ca4074055bce7ca87d8440bc7-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/7c6d529ca4074055bce7ca87d8440bc7-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":306,"real_height_2x":614},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/7c6d529ca4074055bce7ca87d8440bc7-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/7c6d529ca4074055bce7ca87d8440bc7-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":306,"real_height_2x":614}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Посмотрим, где лежат названия разделов. Нажмем правой кнопкой мыши, выберем «просмотреть код».&nbsp;Мы видим, что название раздела лежит в теге span, который вложен в тег a.&nbsp;","type":"p","className":""},{"data":{"id":3266,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/7353708e523a4220aba8ede532de8e0b-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/7353708e523a4220aba8ede532de8e0b-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":662,"real_height_2x":1325},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/7353708e523a4220aba8ede532de8e0b-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/7353708e523a4220aba8ede532de8e0b-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":662,"real_height_2x":1325}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Обратимся к нему с помощью уже известной команды response.css. Если какой-то тег вложен в другой, мы можем дописать его через пробел. Мы дописываем тег span. Слово text поможет увидеть, что именно находится в css-селекторе, к которому мы обращаемся: response.css('a.breadcrumbs__link.smartLink span::text')&nbsp;","type":"p","className":""},{"data":"Мы получили на выход список css-selector, в которых видим нужные слова:&nbsp;","type":"p","className":""},{"data":"<li>Главная</li><li>Художественная литература</li><li>Проза</li><li>Классическая зарубежная литература</li>","type":"ul","className":""},{"data":{"id":3267,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/52300f7fcb8844b2bcac6b0a82e1e42d-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/52300f7fcb8844b2bcac6b0a82e1e42d-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":349,"real_height_2x":698},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/52300f7fcb8844b2bcac6b0a82e1e42d-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/52300f7fcb8844b2bcac6b0a82e1e42d-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":349,"real_height_2x":698}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Нам нужно достать каждый раздел по отдельности. Так как это список, обратимся по номеру элемента. Художественная литература на втором месте после раздела главная. Так как в программировании отсчет идет с нуля, обратимся к первому элементу.","type":"p","className":""},{"data":"Напишем: response.css('a.breadcrumbs__link.smartLink span::text')[1]. На выход мы получим нужный css-селектор.&nbsp;","type":"p","className":""},{"data":"Чтобы выбрать содержание селектора, допишем get(): response.css('a.breadcrumbs__link.smartLink span::text')[1].get().","type":"p","className":""},{"data":"Пробелы в строке по бокам уберем с помощью известного метода strip(): response.css('a.breadcrumbs__link.smartLink span::text')[1].get().strip().","type":"p","className":""},{"data":{"id":3268,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/df6a03f394b74d0f92a8490b632be410-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/df6a03f394b74d0f92a8490b632be410-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":151,"real_height_2x":302},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/df6a03f394b74d0f92a8490b632be410-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/df6a03f394b74d0f92a8490b632be410-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":151,"real_height_2x":302}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Мы получили нужный элемент, добавим его в словарь. Ключ назовем «first_type».","type":"p","className":""},{"data":{"id":3269,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/7022f140efff49178f38458a8ea5bbc4-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/7022f140efff49178f38458a8ea5bbc4-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":220,"real_height_2x":439},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/7022f140efff49178f38458a8ea5bbc4-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/7022f140efff49178f38458a8ea5bbc4-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":220,"real_height_2x":439}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Чтобы получить следующий раздел, к которому относится книга, обратимся ко второму элементу списка: response.css('a.breadcrumbs__link.smartLink span::text')[2].get().strip()","type":"p","className":""},{"data":"Назовем ключ «second_type»: 'second_type': response.css('a.breadcrumbs__link.smartLink span::text')[2].get().strip(). И так же достанем третий последний раздел. ","type":"p","className":""},{"data":{"id":3271,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/50d7555b363045b8942f3b3a8040c09c-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/50d7555b363045b8942f3b3a8040c09c-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":235,"real_height_2x":470},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/50d7555b363045b8942f3b3a8040c09c-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/50d7555b363045b8942f3b3a8040c09c-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":235,"real_height_2x":470}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Мы указали все элементы, которые нам нужно достать. Выходим из ScrapyShell — control+D","type":"p","className":""},{"data":"Запись в файл","type":"h2","className":""},{"data":"Нам даже не нужно прописывать переход по страницам, потому что паук сам соберет все ссылки на сайте, которые соответствуют нашему правилу.&nbsp;Сохраняем файл и можем запускать паука. Результат запишем в csv-файл all_books.csv. Напишем команду в терминале: scrapy crawl all_books -O all_books.csv","type":"p","className":""},{"data":"Паук начал собирать информацию обо всех книгах на сайте. Это займет около 10 минут.&nbsp;В папке с проектом мы видим новый документ. В нем есть информация о каждой из 16 тысяч книг.&nbsp;","type":"p","className":""},{"data":{"id":3273,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/2eb53461df21428ab4fe91bde0378c49-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/2eb53461df21428ab4fe91bde0378c49-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":612,"real_height_2x":1226},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/2eb53461df21428ab4fe91bde0378c49-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/2eb53461df21428ab4fe91bde0378c49-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":612,"real_height_2x":1226}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Это уже полноценный датасет, который можно анализировать. Например, посмотреть, какую книгу покупали чаще всего.&nbsp;Попробуйте выделить пятерку самых популярных книг, результат получится неожиданный.&nbsp;","type":"p","className":""}],"created_at_date":"2021-11-26","updated_at_date":null,"created_at_date_time":null,"updated_at_date_time":null,"is_exclusive":false,"is_urgent":false,"initially_published_at":"2021-11-26T06:55:24.483Z","has_additional_facts":false,"locale":"ru","has_visualization_without_fallback":false,"spec_project":null,"header_secondary":"","warning":"","is_translated_by_ai":false,"with_donation_banner":true,"donation_banner_header":"","donation_banner_text":"","media_outlet__codename":null,"with_authors_request_banner":false,"faq":[],"_path":"/workshops/2021/11/26/parsing-s-pomoshchyu-python-urok-4/","is_published":true,"authors_list":[{"id":23,"lead":"Автор дата-отдела","_path":"polina-uzhvak","avatar":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-896x504px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-896x504px_2x.webp","for_width":896,"for_height":504,"real_width_1x":896,"real_width_2x":1792,"real_height_1x":504,"real_height_2x":1008},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-896x504px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-896x504px_2x.jpg","for_width":896,"for_height":504,"real_width_1x":896,"real_width_2x":1792,"real_height_1x":504,"real_height_2x":1008},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-688x387px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-688x387px_2x.webp","for_width":688,"for_height":387,"real_width_1x":688,"real_width_2x":1376,"real_height_1x":387,"real_height_2x":774},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-688x387px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-688x387px_2x.jpg","for_width":688,"for_height":387,"real_width_1x":688,"real_width_2x":1376,"real_height_1x":387,"real_height_2x":774},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-416x234px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-416x234px_2x.webp","for_width":416,"for_height":234,"real_width_1x":416,"real_width_2x":832,"real_height_1x":234,"real_height_2x":468},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-416x234px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-416x234px_2x.jpg","for_width":416,"for_height":234,"real_width_1x":416,"real_width_2x":832,"real_height_1x":234,"real_height_2x":468},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-336x189px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-336x189px_2x.webp","for_width":336,"for_height":189,"real_width_1x":336,"real_width_2x":672,"real_height_1x":188,"real_height_2x":378},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-336x189px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/a2c3a2eaa4af438c94e0f73b51cc6e26-336x189px_2x.jpg","for_width":336,"for_height":189,"real_width_1x":336,"real_width_2x":672,"real_height_1x":188,"real_height_2x":378}],"lead_en":"","ordering":null,"position":"Автор","is_public":true,"last_name":"Ужвак","first_name":"Полина","position_en":null,"last_name_en":"Uzhvak","first_name_en":"Polina"}],"external_authors_list":[],"editors_list":[],"external_editors_list":[],"participants_list":[],"external_participants_list":[],"partners_list":[],"case":null,"translated_material":null,"audio_material":null,"text_material":null,"id":381,"topics":[],"meta":{"og_title":" ","og_description":"","twitter_title":" ","twitter_description":"","og_image":"https://static.istories.media/uploaded/images/share/eb9a073b35314b44a14859378316b726.jpg","twitter_image":"https://static.istories.media/uploaded/images/share/eb9a073b35314b44a14859378316b726.jpg","vk_image":"https://static.istories.media/uploaded/images/share/vk/5e9690d31b44465a871eba15fab6f948.jpg"},"tags":["scrapy","python","мастерская","парсинг","скрейпинг"],"playlist":null,"socialMediaBanners":[{"id":10,"text":"У пропаганды есть миллиарды из бюджета. У нас — только вы","link":"","codename":"donation"},{"id":14,"text":"Список погибших и пропавших без вести российских военных","link":"","codename":"haron"},{"id":18,"text":"","link":"","codename":"mobile_app"},{"id":6,"text":"Оставайтесь на связи: подпишитесь на нашу рассылку","link":"","codename":"newsletter"}]}},"staticQueryHashes":["764694655"],"slicesMap":{}}