Log Delivery Service

1{
2  "code": 0,
3  "download_url": "http://cs-download.limelight.com/<path to file>",
4  "message": "success",
5  "preview_url": "http://cs-download.limelight.com/<path to file>",  
6}

1import jsonrpclib
2import requests
3
4url = 'http://{Account name}.upload.llnw.net/jsonrpc'
5api = jsonrpclib.Server( url )
6res = api.login(yourUser, yourPassword, True)
7token = res[0]
8
9'''
10User-defined variables
11'''
12storage_log_dir = '/{account name}/_livelogs/'
13pageSize = 10000  # page size for listing log files
14files_to_download = []  # log files to download
15media_vault_expiry = 60  # expiry time for mediaVaultUrl request
16mv_errors = {-1: "Internal error", -2: "Path exists and is a directory", -8: "Invalid path",
17            -34: "Invalid expiry", -60: "Service is disabled or unavailable", -10001: "Invalid token"}
18'''
19Function to examine files returned from calls to listFile
20Based on a condition that you determine, you write file names to a list
21of files that will later be downloaded.
22This simple example looks for file names that contain the number 2.
23'''
24def parse_list(file_list):
25    for (log_file) in file_list:
26    name = log_file['name']
27    if name.find('2') > -1:
28        files_to_download.append(name)
29        print(log_file['name'])
30
31'''
32List Log files. This is a simplistic approach for demonstration purposes.
33Customers might want to try a multi-threaded approach because the number of files can be quite large
34'''
35results = api.listFile(token, storage_log_dir, pageSize, 0, True)
36file_list = results['list']
37if len(file_list) > 0:
38    parse_list(file_list)
39    cookie = results['cookie']
40    while cookie > 0:
41        results = api.listFile(token, storage_log_dir, pageSize, cookie, True)
42        file_list = results['list']
43        parse_list(file_list)
44        cookie = results['cookie']
45
46'''
47Download file. This is a simplistic approach for demonstration purposes.
48Customers might want to try a multi-threaded approach for a large number of files to download.
49'''
50for file_name in files_to_download:
51    log_path = storage_log_dir + '/' + file_name
52    mvu = api.mediaVaultUrl(token, log_path, media_vault_expiry)
53    if mvu['code'] != 0:
54        print("Error attempting to call 'mediaVaultUrl.\nCode: " + str(mvu['code']) + ": " + mv_errors[mvu['code']])
55        mv_download_url = mvu['download_url']
56        # Use the requests library to make the download
57        response = requests.get(mv_download_url)
58        # Upon non-success write a line to your errors file
59        if response.status_code != 200:
60            print("Unable to download " + file_name + ". Status code: " + response.status_code)

1#!/usr/bin/env python
2import logging, sys, csv, itertools
3from multiprocessing import Pool
4import requests
5import time
6import json
7import threading
8
9'''
10Author: spandey
11Unofficial Sample. Provided AS IS. WITHOUT ANY WARRANTY OR CONDITIONS.
12Uses Python 3
13'''
14LOG_FILENAME = 'LDSDownloadSession.log'
15FileList = 'DLFiles.log'
16logging.basicConfig(filename = LOG_FILENAME, level = logging.DEBUG, format='%(asctime)s %(levelname)s-%(filename)s:%(message)s')
17logger = logging.getLogger(__name__)
18
19class StatError(RuntimeError):
20        def __init__(self, arg):
21                self.args = arg
22
23class ListPathError(RuntimeError):
24        def __init__(self, arg):
25                self.args = arg
26
27jsonRPC_Endpoint=''
28token=''
29cookie = ''
30numFiles = 0
31
32numDirs = 0
33totalBytes = 0
34oldFileList = []
35theFileList = []
36dirList = []
37threads = []
38#max files to download per session set to 0 for unlimited download
39max_files = 5
40'''
41User-defined variables
42'''
43storage_log_dir = '/_livelogs/http/<change with the base path>'
44pageSize = 10000  # page size for listing log files
45files_to_download = []  # log files to download
46media_vault_expiry = 60  # expiry time for mediaVaultUrl request
47mv_errors = {-1: "Internal error", -2: "Path exists and is a directory", -8: "Invalid path",
48            -34: "Invalid expiry", -60: "Service is disabled or unavailable", -10001: "Invalid token"}
49'''
50Function to examine files returned from calls to listFile
51Based on a condition that you determine, you write file names to a list
52of files that will later be downloaded.
53This simple example looks for file names that contain the number 2.
54need to add {"method":"deleteFile","id":39,"jsonrpc":"2.0","params":{"token":"a4660cbde39c4dceb2cf796d494db3da","path":"/lll/1.mp4"}}
55'''
56
57def parse_list(file_list):
58    for log_file in file_list:
59        name = log_file['name']
60    if name.find('2') > -1:
61        files_to_download.append(name)
62        print(log_file['name'])
63
64def getFileListing(token, _dirname_, res):
65    numDirs = len(res['dirs'])
66    numFiles =  len(res['files'])
67    _directories_ = res['dirs']
68    print ("Total directory count: " + str(numDirs))
69    print ("Total file count: " + str(numFiles))
70    #Delete the dir in case is empty and is not the base path
71    if numDirs == 0 and numFiles == 0 and _dirname_.count('/') > 3:
72        delp = '{"method":"deleteDir","id":1,"jsonrpc":"2.0","params":{"token":"'+ token +'","path":"'+ _dirname_ +'"}}'
73        print("\nDeleting : "+ delp)
74        delpRes = requests.post(jsonRPC_Endpoint, data=delp)
75        delpRes = json.loads(delpRes.text)
76        delpCode = delpRes['result']
77        #print("\n\n-------------- Code: " + str(delpCode) )
78        if delpCode != 0:
79            print("Error attempting to call del url.\nCode: " + str(delpCode))
80
81    for _dir_ in _directories_:
82        #print ("Scanning Directory: " + _dir_['name'] + " for dirs")
83        dirName = _dirname_ + '/' + _dir_['name']
84        listPath(token, dirName)
85    # Listing Files
86    file_listing = res['files']
87    conteggio = 0
88    for file in file_listing:
89        '''
90        Download file. This is a single-threaded approach for simple use & demonstration purposes.
91        Customers might want to try a multi-threaded approach for a large number of files to download.
92        '''
93        conteggio += 1
94        log_path = _dirname_+ '/' + file['name']
95        #print("\nstarting download of: "+ log_path)
96        mvu = '{"method": "mediaVaultUrl", "id": 1, "params": {"token":"'+ token +'", "path": "'+ log_path +'", "expiry": '+str(media_vault_expiry)+'}, "jsonrpc": "2.0"}'
97        mvuRes = requests.post(jsonRPC_Endpoint, data=mvu)
98        mvuRes = json.loads(mvuRes.text)
99        #print("==== Printing mediaVaultUrl response  ====\n")
100        #print(mvuRes)
101        code = mvuRes['result']['code']
102        if code !=0:
103            print("Error attempting to call 'mediaVaultUrl.\nCode: " + str(code) + ": " + mv_errors[code])
104        else:
105            mv_download_url = mvuRes['result']['download_url']
106            # grab the name of the file to write from mv url
107            lds_file_name = mv_download_url.rsplit("/",1)[1].split("?")[0]
108            print(mv_download_url, '\nFilename:'+lds_file_name)
109            with open(lds_file_name, "wb") as f:
110                # Use the requests library to make the download
111                response = requests.get(mv_download_url, stream=True)
112                # check & show download progress
113                total_length = response.headers.get('content-length')
114                if total_length is None: # no content length header
115                    print("no content-length header found")
116                    f.write(response.content)
117                else:
118                    dl = 0
119                    total_length = int(total_length)
120                    for data in response.iter_content(chunk_size=4096):
121                        dl += len(data)
122                        f.write(data)
123                        done = int(50 * dl / total_length)
124                        sys.stdout.write("\r[%s%s]" % ('|' * done, ' ' * (50-done)) )
125                        sys.stdout.flush()
126                    #Delete the file just downloaded
127                    delu = '{"method":"deleteFile","id":1,"jsonrpc":"2.0","params":{"token":"'+ token +'","path":"'+ log_path +'"}}'
128                    print("\nDeleting : "+ delu)
129                    deluRes = requests.post(jsonRPC_Endpoint, data=delu)
130                    deluRes = json.loads(deluRes.text)
131                    #delcode = deluRes['result']['code']
132                    #print("\n\n-------------- Code: " + str(delcode) + ": " + mv_errors[delcode])
133                    #if delcode !=0:
134                    #    print("Error attempting to call del url.\nCode: " + str(delcode) + ": " + mv_errors[delcode])
135                # Upon non-success write a line to your errors file
136                if response.status_code != 200:
137                    print("Unable to download " + file['name'] + ". Status code: " + response.status_code)
138        if conteggio == max_files:
139            break    # break here
140
141# def writeToFile(filename, data):
142#       with open(filename, 'w') as f:
143#               i = 0
144#               for i in range(len(data)):
145#                       print(data[i]+'\n', end="", file=f)
146
147def listPath(token, _dirname_):
148        '''
149    Listing path recursively
150    '''
151        try:
152                # Scan through parent directory for the files and sub-dirs
153                listpathdata='{"method": "listPath","id": 1,"params": {"token": "'+token+'","path": "'+_dirname_+'","pageSize": '+str(pageSize)+',"cookie": "'+cookie+'","stat": true},"jsonrpc": "2.0"}'
154                # print('===List Path Data - Parent==\n', listpathdata)
155                res = requests.post(jsonRPC_Endpoint, data=listpathdata)
156                res = json.loads(res.text)
157                print ('======Listing Path for: '+_dirname_)
158                code = res['result']['code']
159
160                if code !=0:
161                        msg = 'Error issuing listPath command on directory: ' + _dirname_
162                        msg += '\n Return code: ' + str( code )
163                        msg += '\n See API documentation for details.'
164                        logger.error('ListPathError' + msg)
165                        raise ListPathError( msg )
166
167                theFileList = getFileListing(token,_dirname_,res['result'])
168
169        except ListPathError as e:
170                print (''.join( e.args ))
171
172        except StatError as e:
173                print (''.join( e.args ))
174
175def main(host, username, password):
176    global jsonRPC_Endpoint
177    jsonRPC_Endpoint = host
178    try:
179        # Obtain token
180        loginData='{"method": "login","id": 0,"params": {"username": "'+username+'","password":"'+password+'","detail": true},"jsonrpc": "2.0"}'
181        login = requests.post(jsonRPC_Endpoint, data=loginData)
182        print (login.reason, login.headers)
183        resp = json.loads(login.text)
184        token = resp['result'][0]
185        print ('=======Token & User======\n',token)
186        logger.debug('Logged In. Token Is: '+ token)
187
188        # Persist Token for the session until the logout or the end of time defined by 'expire'
189        persistData='{"method": "updateSession","id": 4,"params": {"token": "'+token+'", "expire": 0},"jsonrpc": "2.0"}'
190        persist = requests.post(jsonRPC_Endpoint, data=persistData)
191        persist = json.loads(persist.text)
192        #print('=======Updated Token======\n',persist)
193        if persist['result'] == 0:
194        #    print('Token Persisted!')
195            logger.debug('Token Persisted!. Token Is: '+ token)
196
197        # call listPath method on storage
198        listPath(token, storage_log_dir)
199    except Exception as e:
200        print(''.join( e.args ))
201        logger.error('Error Occured While Logging')
202    finally:
203        #print ('\nLogging out...\n')
204        logoutData = '{"method": "logout","id": 1,"params": {"token": "'+token+'"},"jsonrpc": "2.0"}'
205        logoutRes = requests.post(jsonRPC_Endpoint, data=logoutData)
206        logoutRes = json.loads(logoutRes.text)
207        #print ('Logout results: ', logoutRes)
208        if logoutRes['result'] == 0:
209            logger.debug('Logged Out!')
210
211if __name__ == "__main__":
212    main('https://<shortname>-l.upload.llnw.net/jsonrpc2', '<user-vs>',"<password-vs>")