The scrapy crawl site always shows that you are not logged in.

the scrapy code is as follows:

-sharp!/usr/bin env python3
import scrapy, json
from scrapy import FormRequest


class jingzhunspider(scrapy.Spider):
    name = "jingzhun"
    allowed_domains = []
    start_urls = ["https://rong.36kr.com/"]

    def __init__(self):
        self.headers = {"Referer": "https://rong.36kr.com/list/detail&?sortField=HOT_SCORE&p=36",
                        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
                        "Cookie": "acw_tc=b65cfd2515395760831792797e7a30fed7278a95d7c68d0dcad0b9cbc4ac1b; kwlo_iv=1h; kr_stat_uuid=CQ5e225664238; Hm_lvt_e8ec47088ed7458ec32cde3617b23ee3=1539833640,1539835270,1539847843,1539854305; Hm_lpvt_e8ec47088ed7458ec32cde3617b23ee3=1539854305; download_animation=1; _kr_p_se=3bc105b4-9a7d-42fb-add8-bf95717e809e; krid_user_id=1174461189; krid_user_version=4; kr_plus_id=1174461189; kr_plus_token=sxXE3YtIANlixxyKQlgRsxUW9bw5_1182767____; device-uid=010004b0-d346-11e8-a96a-4199591fdd87; Hm_lvt_713123c60a0e86982326bae1a51083e1=1539836728,1539847818,1539915819,1540172654; Hm_lvt_1684191ccae0314c6254306a8333d090=1539836728,1539847818,1539915819,1540172655; kr_plus_utype=0; krnewsfrontss=4963ea68949936658d715f1da01ac2dc; M-XSRF-TOKEN=82688f4b00348470d9150fa6723dd91d4ef3037d93ccf1c010c11e523f0c4d80; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221174461189%22%2C%22%24device_id%22%3A%221668a22a82493-063c5cb7bcafff-163b6953-1296000-1668a22a82533c%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%2C%22first_id%22%3A%221668a22a82493-063c5cb7bcafff-163b6953-1296000-1668a22a82533c%22%7D; Hm_lpvt_713123c60a0e86982326bae1a51083e1=1540189058; Hm_lpvt_1684191ccae0314c6254306a8333d090=1540189059; Z-XSRF-TOKEN=eyJpdiI6IkNoUVM5ZUxnSTZlUkhTQlNkMmtzdXc9PSIsInZhbHVlIjoiUG11V2xcLzZXZElMWjRBY3l1MWxvc3ZFOWQ4S1pRNXd2b0FJSUk3Y01ZRHJSV1d3OEd3N25zcUNRaWFRYVpvenI0WmNlZmt4VStrU0NMYnc3XC94cklzUT09IiwibWFjIjoiYWY2OTFlNDA4NzNhYjAwYjg4MmZlYWZmNTljNmY4ZjY3ZTFhNjkwNzcxM2Y0MmQ5MDBlOTg5NzMwYjJlMjE3YiJ9; krchoasss=eyJpdiI6IjZJNXE4T0lkVXhMdjAxXC9OeDh3OUp3PT0iLCJ2YWx1ZSI6IjBQUm1cL1lQS2RpUFRmdndLN3ExYkdXdXVjSE9HY1RFYVB0NEh0S3VwbHhLMUczQlJ3RHdmMXVldVRZZ0JUeGVSWjY2a1R1QjZkMWd1RFdja2RBaTU1QT09IiwibWFjIjoiZWQzMmJmNDNkZTYzMTIwZTQwZGU1ZDdkM2EyN2Q2YWRiMzg2YmQ4ZGIyZmE4MjI1YjgxZGI0YzY2YzMzMzAxMSJ9"
            ,"Host": "rong.36kr.com"}

    def start_requests(self):
        yield scrapy.Request(url="https://rong.36kr.com/n/api/column/0/company?sortField=HOT_SCORE&p=38",callback=self.get_info, headers=self.headers, dont_filter=True)

    def get_info(self, response):
        print(response.text)  
:
18-10-22 17:32:00 [scrapy.middleware] INFO: Enabled downloader middlewares:
["scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware",
 "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware",
 "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware",
 "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware",
 "scrapy.downloadermiddlewares.retry.RetryMiddleware",
 "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware",
 "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware",
 "scrapy.downloadermiddlewares.redirect.RedirectMiddleware",
 "scrapy.downloadermiddlewares.cookies.CookiesMiddleware",
 "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware",
 "scrapy.downloadermiddlewares.stats.DownloaderStats"]
2018-10-22 17:32:00 [scrapy.middleware] INFO: Enabled spider middlewares:
["scrapy.spidermiddlewares.httperror.HttpErrorMiddleware",
 "scrapy.spidermiddlewares.offsite.OffsiteMiddleware",
 "scrapy.spidermiddlewares.referer.RefererMiddleware",
 "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware",
 "scrapy.spidermiddlewares.depth.DepthMiddleware"]
2018-10-22 17:32:00 [scrapy.middleware] INFO: Enabled item pipelines:
[]
2018-10-22 17:32:00 [scrapy.core.engine] INFO: Spider opened
2018-10-22 17:32:00 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2018-10-22 17:32:00 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
2018-10-22 17:32:00 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://rong.36kr.com/n/api/column/0/company?sortField=HOT_SCORE&p=38> (referer: https://rong.36kr.com/list/detail&?sortField=HOT_SCORE&p=36)
{
    "code": 403,
    "data": "GET https://rong.36kr.com/n/api/column/0/company?sortField=HOT_SCORE&p=38",
    "msg": " "
}

,


 import requests
headers={"Referer":"https://rong.36kr.com/list/detail&?sortField=HOT_SCORE","User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36","Cookie":"acw_tc=b65cfd2515395760831792797e7a30fed7278a95d7c68d0dcad0b9cbc4ac1b; kwlo_iv=1h; kr_stat_uuid=CQ5e225664238; Hm_lvt_e8ec47088ed7458ec32cde3617b23ee3=1539833640,1539835270,1539847843,1539854305; Hm_lpvt_e8ec47088ed7458ec32cde3617b23ee3=1539854305; download_animation=1; _kr_p_se=3bc105b4-9a7d-42fb-add8-bf95717e809e; krid_user_id=1174461189; krid_user_version=4; kr_plus_id=1174461189; kr_plus_token=sxXE3YtIANlixxyKQlgRsxUW9bw5_1182767____; device-uid=010004b0-d346-11e8-a96a-4199591fdd87; Hm_lvt_713123c60a0e86982326bae1a51083e1=1539836728,1539847818,1539915819,1540172654; Hm_lvt_1684191ccae0314c6254306a8333d090=1539836728,1539847818,1539915819,1540172655; kr_plus_utype=0; krnewsfrontss=4963ea68949936658d715f1da01ac2dc; M-XSRF-TOKEN=82688f4b00348470d9150fa6723dd91d4ef3037d93ccf1c010c11e523f0c4d80; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221174461189%22%2C%22%24device_id%22%3A%221668a22a82493-063c5cb7bcafff-163b6953-1296000-1668a22a82533c%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%2C%22first_id%22%3A%221668a22a82493-063c5cb7bcafff-163b6953-1296000-1668a22a82533c%22%7D; Hm_lpvt_713123c60a0e86982326bae1a51083e1=1540189058; Hm_lpvt_1684191ccae0314c6254306a8333d090=1540189059; Z-XSRF-TOKEN=eyJpdiI6IkNoUVM5ZUxnSTZlUkhTQlNkMmtzdXc9PSIsInZhbHVlIjoiUG11V2xcLzZXZElMWjRBY3l1MWxvc3ZFOWQ4S1pRNXd2b0FJSUk3Y01ZRHJSV1d3OEd3N25zcUNRaWFRYVpvenI0WmNlZmt4VStrU0NMYnc3XC94cklzUT09IiwibWFjIjoiYWY2OTFlNDA4NzNhYjAwYjg4MmZlYWZmNTljNmY4ZjY3ZTFhNjkwNzcxM2Y0MmQ5MDBlOTg5NzMwYjJlMjE3YiJ9; krchoasss=eyJpdiI6IjZJNXE4T0lkVXhMdjAxXC9OeDh3OUp3PT0iLCJ2YWx1ZSI6IjBQUm1cL1lQS2RpUFRmdndLN3ExYkdXdXVjSE9HY1RFYVB0NEh0S3VwbHhLMUczQlJ3RHdmMXVldVRZZ0JUeGVSWjY2a1R1QjZkMWd1RFdja2RBaTU1QT09IiwibWFjIjoiZWQzMmJmNDNkZTYzMTIwZTQwZGU1ZDdkM2EyN2Q2YWRiMzg2YmQ4ZGIyZmE4MjI1YjgxZGI0YzY2YzMzMzAxMSJ9"
,"Host":"rong.36kr.com"}
res=requests.get("https://rong.36kr.com/n/api/column/0/company?sortField=HOT_SCORE&p=1",headers=headers)
print(res.text)

I can return the data myself, but not with the framework

Why is this

Python

Sep.10,2021

cookie={'acw_tc': 'b65cfd2515395760831792797e7a30fed7278a95d7c68d0dcad0b9cbc4ac1b', 'kwlo_iv': '1h', 'kr_stat_uuid': 'CQ5e225664238', 'Hm_lvt_e8ec47088ed7458ec32cde3617b23ee3': '1539833640,1539835270,1539847843,1539854305', 'Hm_lpvt_e8ec47088ed7458ec32cde3617b23ee3': '1539854305', 'download_animation': '1', '_kr_p_se': '3bc105b4-9a7d-42fb-add8-bf95717e809e', 'krid_user_id': '1174461189', 'krid_user_version': '4', 'kr_plus_id': '1174461189', 'kr_plus_token': 'sxXE3YtIANlixxyKQlgRsxUW9bw5_1182767____', 'device-uid': '010004b0-d346-11e8-a96a-4199591fdd87', 'Hm_lvt_713123c60a0e86982326bae1a51083e1': '1539836728,1539847818,1539915819,1540172654', 'Hm_lvt_1684191ccae0314c6254306a8333d090': '1539836728,1539847818,1539915819,1540172655', 'kr_plus_utype': '0', 'krnewsfrontss': '4963ea68949936658d715f1da01ac2dc', 'M-XSRF-TOKEN': '82688f4b00348470d9150fa6723dd91d4ef3037d93ccf1c010c11e523f0c4d80', 'sensorsdata2015jssdkcross': '%7B%22distinct_id%22%3A%221174461189%22%2C%22%24device_id%22%3A%221668a22a82493-063c5cb7bcafff-163b6953-1296000-1668a22a82533c%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%2C%22first_id%22%3A%221668a22a82493-063c5cb7bcafff-163b6953-1296000-1668a22a82533c%22%7D', 'Hm_lpvt_713123c60a0e86982326bae1a51083e1': '1540189058', 'Hm_lpvt_1684191ccae0314c6254306a8333d090': '1540189059', 'Z-XSRF-TOKEN': 'eyJpdiI6IkNoUVM5ZUxnSTZlUkhTQlNkMmtzdXc9PSIsInZhbHVlIjoiUG11V2xcLzZXZElMWjRBY3l1MWxvc3ZFOWQ4S1pRNXd2b0FJSUk3Y01ZRHJSV1d3OEd3N25zcUNRaWFRYVpvenI0WmNlZmt4VStrU0NMYnc3XC94cklzUT09IiwibWFjIjoiYWY2OTFlNDA4NzNhYjAwYjg4MmZlYWZmNTljNmY4ZjY3ZTFhNjkwNzcxM2Y0MmQ5MDBlOTg5NzMwYjJlMjE3YiJ9', 'krchoasss': 'eyJpdiI6IjZJNXE4T0lkVXhMdjAxXC9OeDh3OUp3PT0iLCJ2YWx1ZSI6IjBQUm1cL1lQS2RpUFRmdndLN3ExYkdXdXVjSE9HY1RFYVB0NEh0S3VwbHhLMUczQlJ3RHdmMXVldVRZZ0JUeGVSWjY2a1R1QjZkMWd1RFdja2RBaTU1QT09IiwibWFjIjoiZWQzMmJmNDNkZTYzMTIwZTQwZGU1ZDdkM2EyN2Q2YWRiMzg2YmQ4ZGIyZmE4MjI1YjgxZGI0YzY2YzMzMzAxMSJ9'} scrapy.Request(url="https://rong.36kr.com/n/api/column/0/company?sortField=HOT_SCORE&p=38",cookies=cookie,callback=self.get_info, headers=self.headers, dont_filter=True)