Python login with request but can't have the complet page

olivier_9248525 0
Viewed 54

It seems that I can't get the page complete with my script, yet I have checked the send request which is normally good. I created a test account for the occasion. here is my script:

import requests
from bs4 import BeautifulSoup as bs

with requests.Session() as c:

    url = 'https://www.siemens-syncoic.com/#/Login'
    USERNAME = "lj@yopmail.com"
    PASSWORD = "Password.01"
    c.get(url)
    login_data = dict(username=USERNAME, password=PASSWORD, remenber='false')
    c.post(url, data=login_data)
    page = c.get('https://www.siemens-syncoic.com/#/Operating', timeout=(2, 5))
    print(page.status_code)
    print(bs(page.content, 'html.parser'))

I got this result:

    200

    <!DOCTYPE html>
    
    <html>
    <head>
    <title></title>
    <meta content="2020-05-19T13:09:14" name="ClimatixICbuild-date-ui"/>
    <meta content="14.0.7444" name="ClimatixICversion-ui"/>
    <meta content="14.0.7444" name="ClimatixICversion-api"/>
    <meta content="14.0.7444" name="ClimatixICversion-connectivity-tls11"/>
    <meta content="14.0.7444" name="ClimatixICversion-connectivity-tls12"/>
    <meta content="14.0.7444" name="ClimatixICversion-connectivity-soc"/>
    <meta content="14.0.7444" name="ClimatixICversion-connectivity-sth"/>
    <meta content="© Siemens AG 2013 - 2020 All Rights Reserved" name="ClimatixICcopyright"/>
    <meta content="width=device-width, initial-scale=1.0" name="viewport"/>
    <script type="text/javascript">
            console.time('load files')
            window.SERVER_CONFIG = {
                APP_ROOT_PATH: '/Frontend/dist/',
                APP_MODE_PROD: true,
                API_URL: 'https://backend.api.climatixic.com' + '/Public',
                DECIMAL_SEPERATOR: '.',
                GROUP_SEPERATOR: ',',
                DEFAULT_LANGUAGE: 'en-US',
                CURRENT_THEME: '',
                MVC_SCRIPTS: [
                    {
                        path: '/bundles/newui?v=3y6l1SmB3IYPNjlDiBaMauJJSTuX-EdgxxZoccswI9o1'
                    },
                    {
                        path: '/bundles/globalize?v=0JjlGHj5TrHXZT3Y5icAMYgxzrQFAnkGr-fjs-u5kPM1'
                    },
                    {
                        path: '/bundles/globalize/en?v=gpUbXiN0legtaAjCFj_qS8lAax3kDu08iVOL-80rCnc1'
                    },
                    {
                        script: 'Globalize.locale("en");'
                    },
                    {
                        path: '/bundles/dxwebappjs?v=vJE5ySakgAF8yjDHMgwCkfgsxA0qNyfpNCRn-zVc67M1'
                    },
                    {
                        path:  '/bundles/dxwebappjs/en-US?v=sTEBg3eGmyhji_Yof9X2ulVuLQEyZOh2tFYErmbVReA1'
                    },
                    {
                        script: 'DevExpress.ui.dxOverlay.baseZIndex(2100);'
                    },
                    {
                        path: '/Frontend/dist/old/old-content.adapter.js?v=14.0.7444'
                    }
                ],
                APP_URL: '',
                SUPPORTED_LANGUAGES : ['en-US','en-GB','de-DE','de-CH','fr-FR','it-IT','nl-NL','pl-PL','sv-SE','fi-FI','nb-NO','da-DK','cs-CZ','sk-SK','es-ES','pt-PT','hu-HU','ro-RO','el-GR','ru-RU','tr-TR','uk-UA','ja-JP','zh-CN','ko-KR'],
                MAP_PROVIDER: 'Google',
                MAP_PROVIDER_KEY: 'AIzaSyCA1pATiXFbmoPO2w5lfjZMn-CsSl0y4LE:90'
                    , EXT_AUTH_TYPE: 'siemensid'
                                                                                                                    , PROXY_SESSION_TIMEOUT: 540000
            };
        </script>
    <link href="/Frontend/dist/css/basic.css" rel="stylesheet"/>
    <script src="/Frontend/dist/scripts/init.js"></script>
    </head>
    <body>
    <div class="loader">
    <div class="spinner">
    <div class="rect1"></div>
    <div class="rect2"></div>
    <div class="rect3"></div>
    <div class="rect4"></div>
    <div class="rect5"></div>
    </div>
    <div class="loader-text">Loading Application</div>
    </div>
    <cl-app></cl-app>
    <script src="/Frontend/dist/runtime-es2015.js?v=14.0.7444"></script>
    <script src="/Frontend/dist/polyfills-es2015.js?v=14.0.7444"></script>
    <script src="/Frontend/dist/main-es2015.js?v=14.0.7444"></script>
    </body>
    </html>

But i should have a lot more html code... It like it loading but therefore i put a timeout=(2,5) on my get()...

python-requests web-scraping python
1 Answers

Your login doesn't work because, you are using wrong URL for login. Use headers while sending request. After login the data you are looking for is load through XHR request.

import requests

headers = {
    'Host': 'www.siemens-syncoic.com',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'en-US,en;q=0.5',
    'Accept-Encoding': 'gzip, deflate, br',
    'Content-Type': 'application/json',
    'Origin': 'https://www.siemens-syncoic.com',
    'Referer': 'https://www.siemens-syncoic.com/'
}

with requests.Session() as session:

    login_page_url = 'https://www.siemens-syncoic.com/#/Login'
    session.get(login_page_url)

    login_url = "https://www.siemens-syncoic.com/clx/authentication"
    login_data = {"password":"Password.01","rememberme":"false","username":"lj@yopmail.com"}
    response = session.post(login_url, json=login_data, headers=headers)

    print('LoginResponse:', response.status_code, response.content)

    payload = {"requireTotalCount":"false","requireGroupCount":"false","sort":"[{\"selector\":\"Tenant\",\"desc\":false},{\"selector\":\"Name\",\"desc\":false}]","filter":"[\"Tenant\",\"=\",\"Siemens+Proxy\"]"}

    page = session.post('https://www.siemens-syncoic.com/dxdata/dxsites', data=payload, timeout=(2, 5))
    print(page.status_code, response.content)
user_107383090
Related