8 files changed, 294 insertions(+), 5 deletions(-)

M .gitignore
M CMakeLists.txt
M config/bootstrap_config
M config/requirements.txt
A doc/download_assets.md
M docker/docker-compose.yml
M in_docker.sh
A tools/download_asset.py

@@ 50,3 50,5 @@ docker/assets/requirements*
 *.swo
 tags
 .ycm_extra_conf.py
+
+update/

@@ 278,6 278,11 @@ if (${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo")
         )
 endif()
 
+add_custom_target(ecoboot.bin 
+    COMMAND ${CMAKE_SOURCE_DIR}/tools/download_asset.py -w ${CMAKE_BINARY_DIR}/update ecoboot download
+    BYPRODUCTS update/ecoboot.bin
+    )
+
 message_serial_status()
 
 option (BUILD_DOC_WITH_ALL "Build documentation" OFF)

@@ 49,6 49,9 @@ INSTALL_PACKAGES="
         wget \
         python3-magic \
         python3-pip \
-        libfdisk-dev
+        libfdisk-dev \
+        python3-requests \
+        libfuse-dev \
+        libblkid-dev 
 "
 

@@ 1,1 1,2 @@
 gitpython==3.1.11
+tqdm==4.54.1

@@ 0,0 1,85 @@
+# Download Assets 
+
+Building update packages requires some external assets.
+For now we need only `ecoboot.bin`.
+This file may be manually downloaded from [ecoboot releases page](https://github.com/mudita/ecoboot/releases),
+from the "Assets" section. This requires the user to log in to GitHub and click inside the web browser, which is a "no go" for automated builds.
+To automate this process we have introduced a tool `tools/download_asset.py`.
+
+## GitHub API token
+
+To use GitHub API your GitHub login and special token (API password) are required.
+This token can by generated on "User" -> [Settings](https://github.com/settings/profile) page in section [Developer settings](https://github.com/settings/apps) -> [Personal acces tokens](https://github.com/settings/tokens) section.
+For more info please refer to the Github help page about adding tokens: [Creating a personal access token](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token)
+### Scopes
+When you generate a token you can set "Scopes" for which the token will work.
+For downloading assets, we need a ":repo" scope with all sub-scopes.
+
+**Note:** Tokens are displayed only during their creation, so this is the only moment you can copy them.
+If you forget to do this, you will have to generate a new token.
+
+## Storing settings
+The `download_assets.py` file can have `login` and `password` passed through parameters,
+or this data can be stored in your current git repository configuration.
+
+Adding GitHub login to:
+```bash
+git config --add user.githublogin <login>
+```
+
+Adding API Token:
+```bash
+git config --add user.apitoken
+```
+
+Checking the values:
+```bash
+git config user.githublogin
+git config user.apitoken
+```
+
+## How to use `download_asset.py`
+
+```bash
+$./tools/download_asset.py --help
+usage: download_asset.py [-h] [-w WORKDIR] [-t TOKEN] [-l LOGIN] repository {list,ll,download,dw} ...
+```
+
+To work properly, `download_asset.py` requires <Login> and <Token>. If they are stored in repo config these
+parameters are not required, otherwise, our tool will protest if parameters are not set.
+If parameters are set in `.git/config` and You still add these values, they will override the ones in the config.
+
+This tool downloads assets only from our "mudita" organisation in GitHub, but we can change the repository,
+this is `repository` parameter, the second required parameter is a task to be completed:
+Tasks:
+* `list` or `ll`     - list releases, first column is a tag, which may be used in download command.
+* `download` or `dw` - downloads first asset in release (if no tag is added, the latest release is used).
+
+## Usage example
+
+List releases:
+```bash
+./tools/download_asset.py ecoboot ll
+ tag | name | date | pre-release
+--------------------------------
+1.0.4 | Bug fixes for checksum handling | 2020-08-27T13:26:11Z | True
+1.0.1 |  | 2020-08-12T09:45:53Z | False
+1.0.3 | 1.0.3 JSON to ini migration | 2020-07-28T13:29:56Z | False
+0.1 | Support for 2 partitions and boot.ini | 2020-07-07T14:48:18Z | False
+05_07_2019 | Working with PurePhone | 2019-07-05T11:36:51Z | True
+```
+
+Download latest `ecoboot.bin`
+```bash
+./tools/download_asset.py ecoboot dw
+```
+
+Download `ecoboot.bin` for tag `1.0.3`
+```bash
+./tools/download_asset.py ecoboot dw 1.0.3
+```
+
+Download the latest MuditaOS package to the `packages` directory with a custom login and API token
+```bash
+./tools/download_asset.py -w packages -l someUser -t secreetToken123123123 MuditaOs dw
+```

@@ 1,21 1,21 @@
 version: '3'
 services:
  gh-runner0:
-   image: wearemudita/mudita_os_builder:1.4
+   image: wearemudita/mudita_os_builder:1.6
    environment:
      WORKER_NAME: PureBuilder0
    env_file:
      - runner_settings
    entrypoint: /cmd.sh
  gh-runner1:
-   image: wearemudita/mudita_os_builder:1.4
+   image: wearemudita/mudita_os_builder:1.6
    environment:
      WORKER_NAME: PureBuilder1
    env_file:
      - runner_settings
    entrypoint: /cmd.sh
  gh-runner2:
-   image: wearemudita/mudita_os_builder:1.4
+   image: wearemudita/mudita_os_builder:1.6
    environment:
      WORKER_NAME: PureBuilder2
    env_file:

@@ 3,7 3,7 @@
 # For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md
 
 CONTAINER_NAME="wearemudita/mudita_os_builder"
-CONTAINER_TAG="1.4"
+CONTAINER_TAG="1.5"
 CONTAINER=${CONTAINER_NAME}:${CONTAINER_TAG}
 PURE_HOME=`pwd`
 STANDARD_OPTIONS="-v `pwd`:${PURE_HOME} --user \"$(id -u):$(id -g)\" --env HOME=${PURE_HOME} -t"

@@ 0,0 1,193 @@
+#!/usr/bin/env python3
+"""
+Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved.
+For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md
+
+Download ecooboot.bin from repository
+
+@package globalFinder
+"""
+
+import argparse
+import git
+import json
+import os
+import requests
+import sys
+from tqdm import tqdm
+
+
+class Getter(object):
+    '''Download latest ecooboot.bin image'''
+
+    def __init__(self):
+        self.host = 'https://api.github.com/repos'
+        self.organisation = 'mudita'
+        self.repo = 'ecoboot'
+        self.restPrefix = self.host
+        self.restPrefix += '/'
+        self.restPrefix += self.organisation
+        self.restPrefix += '/'
+        self.restPrefix += self.repo
+        self.restPrefix += '/'
+        self.apitoken = None
+        self.ghLogin = None
+        self.getGitRoot()
+        self.getApiToken()
+        self.getGHLogin()
+        self.releases = []
+        self.workdir = ""
+
+    def getGitRoot(self):
+        'Find git root directory'
+        self.gitRepo = git.Repo(os.getcwd(), search_parent_directories=True)
+        self.gitRoot = self.gitRepo.git.rev_parse("--show-toplevel")
+        return self.gitRoot
+
+    def getApiToken(self, args=None):
+        if args is not None:
+            if args.token is not None:
+                self.apitoken = args.token
+                return
+        try:
+            gitConfigReader = self.gitRepo.config_reader()
+            self.apitoken = gitConfigReader.get_value("user", "apitoken")
+        except git.exc.NoOptionError as error:
+            pass
+
+    def getGHLogin(self, args=None):
+        if args is not None:
+            if args.login is not None:
+                self.ghLogin = args.login
+                return
+        try:
+            gitConfigReader = self.gitRepo.config_reader()
+            self.ghLogin = gitConfigReader.get_value("user", "githublogin")
+        except git.exc.NoOptionError as error:
+            pass
+
+    def findWorkDir(self):
+        '''Finds and sets current workdir (based on 'workdir' argument)'''
+        if os.path.isabs(self.args.workdir):
+            self.workdir = self.args.workdir
+        else:
+            self.workdir = self.getGitRoot() + "/" + self.args.workdir
+
+    def getReleases(self, args):
+        '''List only releases'''
+        self.args = args
+        self.getGHLogin(args)
+        self.getApiToken(args)
+        request = self.restPrefix + "releases"
+        headers = {'accept': 'application/vnd.github.v3+json'}
+        page = 0
+        itemsOnPage = 100
+        count = itemsOnPage
+        while count == itemsOnPage:
+            page += 1
+            queryParams = {'per_page': itemsOnPage, 'page': page}
+            response = requests.get(request, auth=(self.ghLogin, self.apitoken), headers=headers, params=queryParams)
+            if response.status_code != requests.codes.ok:
+                print("download error:", response.status_code)
+                print(response.content)
+                sys.exit(1)
+            items = json.loads(response.content)
+            count = len(items)
+            self.releases += items
+        self.releases.sort(key=lambda r: r['published_at'], reverse=True)
+
+    def listReleases(self, args):
+        self.getReleases(args)
+        print(" tag | name | date | pre-release")
+        print("--------------------------------")
+        for release in self.releases:
+            print(release['tag_name'], release['name'], release['published_at'], release['prerelease'], sep=" | ")
+
+    def downloadRelease(self, args):
+        print(sys._getframe().f_code.co_name)
+        self.getReleases(args)
+        print(args.tag)
+        release = None
+        if args.tag is None:
+            release = self.releases[0]
+        else:
+            for rel in self.releases:
+                if rel['tag_name'] == args.tag:
+                    release = rel
+                    break
+        if release is None:
+            print("No release with tag:", args.tag)
+        assets = release['assets']
+        self.downloadAsset(assets[0])
+
+    def downloadAsset(self, asset):
+        self.createWorkdir()
+        print("name:", asset['name'])
+        print("url:", asset['url'])
+        print("size:", asset['size'])
+        headers = {'accept': 'application/octet-stream'}
+        response = requests.get(asset['url'],
+                                auth=(self.ghLogin, self.apitoken),
+                                headers=headers,
+                                stream=True)
+        progres_bar = tqdm(total=asset['size'], unit='iB', unit_scale=True)
+        with open(self.args.workdir + "/" + asset['name'], 'wb') as fd:
+            for chunk in response.iter_content(chunk_size=1024):
+                progres_bar.update(len(chunk))
+                fd.write(chunk)
+        if response.status_code != requests.codes.ok:
+            print("download error:", response.status_code)
+            print(response.content)
+            sys.exit(2)
+
+    def createWorkdir(self):
+        from pathlib import Path
+        Path(self.workdir).mkdir(parents=True, exist_ok=True)
+
+    def run(self, args):
+        self.args = args
+        self.findWorkDir()
+        print(self.args)
+
+
+def main():
+    getter = Getter()
+
+    parser = argparse.ArgumentParser(description="Download ecooboot")
+    parser.add_argument('-w', '--workdir', help="Directory where package is build", default="update")
+    parser.add_argument('-t', '--token',
+                        help="GitHub security token "
+                             "by default read from `git config user.apitoken` "
+                             "Generate it on https://github.com/settings/tokens "
+                             "Set privileges to `repo` "
+                             "and store with command: `git config -add user.apitoken <token from GH>",
+                        required=(getter.apitoken is None))
+    parser.add_argument('-l', '--login',
+                        help="GitHub login"
+                             "by default read from `git config user.githublogin`"
+                             "to add your login use:"
+                             "`git config -add user.githublogin <login>`",
+                        required=(getter.ghLogin is None))
+
+    parser.add_argument('repository', help="repository name from which assets will be downloaded")
+
+    parser.set_defaults(func=getter.run)
+    subparsers = parser.add_subparsers(title="commands",
+                                       description="available commands")
+    listReleases_args = subparsers.add_parser('list', aliases=['ll'],
+                                              description="List public releases")
+    listReleases_args.set_defaults(func=getter.listReleases)
+
+    getReleases_args = subparsers.add_parser('download', aliases=['dw'],
+                                             description="Download Release based on tag or the latest")
+    getReleases_args.set_defaults(func=getter.downloadRelease)
+    getReleases_args.add_argument('tag', help="Download release with selected tag", nargs='?')
+
+    args = parser.parse_args()
+    getter.repo = args.repository
+    getter.workdir = args.workdir
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()