Commit 04e00a57 authored by Laurent Heirendt's avatar Laurent Heirendt
Browse files

Merge branch 'linkchecker' into 'develop'

add linkchecker

See merge request !286
parents 937d88ea 13115127
Pipeline #45838 passed with stages
in 1 minute and 53 seconds
import os, re
from os import path
from natsort import natsorted
from pathlib import Path
def line_prepender(filename, line):
with open(filename, 'r+') as f:
......@@ -100,6 +101,7 @@ sections = natsorted(sections)
# Index contains the generated content, init it with an empty container
index = ''
index += '\n<div class="index-box-container">\n'
whiteList = ''
localIndexArr = [[]] * len(sections)
for folder in cardDirs:
......@@ -157,6 +159,12 @@ for folder in cardDirs:
header += " - /" + folder + "/cards/" + shortcut + "\n"
header += "---"
# add autogenerated links to whitelist
whiteList += permalink + "\n"
whiteList += "/?" + shortcut + "\n"
whiteList += "/cards/" +shortcut + "\n"
whiteList += "/" + folder + "/cards/" + shortcut + "\n"
# add the header properly speaking
line_prepender(fileName, header)
......@@ -197,7 +205,7 @@ for s in sections:
index += build_section_end()
k += 1
# Close the container
# close the container
index += "\n</div>"
## add link to return to main index
......@@ -226,3 +234,13 @@ with open(indexFile, 'w') as file:
file.write(filedata)
print("\n > New index generated and saved in " + indexFile)
# write link whitelist out
whiteListFile = ".ci/whitelist.txt"
if Path(whiteListFile).exists():
with open(whiteListFile, 'r') as file :
for line in file:
whiteList += line
with open(whiteListFile, 'w') as file:
file.write(whiteList)
/privacy-policy
/search
https://www.dev47apps.com/
https://www.dev47apps.com/droidcam/connect/
https://cerbere.uni.lu/
http://iptel.uni.lux/
javascript:%20showBanner();
javascript:showBanner();
media/7zip-encryption-windows.mp4?width=400
media/Get_MD5_checksum_windows.mp4
......@@ -42,4 +42,6 @@ handbook/
internal/handbook
internal/handbook-additional
internal/handbook-annexes
!internal/covid-19/exit-strategy/*.pdf
\ No newline at end of file
!internal/covid-19/exit-strategy/*.pdf
checkFolder
.cache/
stages:
- prepare
- save
- build
- check
- generate
- deploy
- trigger
......@@ -26,6 +29,7 @@ prepare:index:
- python .ci/generateIndex.py
- mkdir .tmp
- cp cards.md .tmp/.
- cp .ci/whitelist.txt .tmp/.
artifacts:
expire_in: 1 day
paths:
......@@ -77,27 +81,49 @@ build:pages:
expire_in: 1 day
paths:
- build
- .tmp
rules:
- if: $CI_COMMIT_REF_NAME
- if: $CI_MERGE_REQUEST_ID
before_script:
- apt-get -qq update
- apt-get install -y -qq git-lfs
- gem install bundler:$BUNDLER_VERSION && bundle install
script:
# Generate the configuration for forks (will use Gitlab Pages on personal namespaces)
- 'echo "url: https://$CI_PROJECT_NAMESPACE.$CI_PAGES_DOMAIN" >> .ci/_config_gitlab_pages.yml'
- 'echo "baseurl: /$CI_PROJECT_NAME" >> .ci/_config_gitlab_pages.yml'
# If there is no config for the current branch, use the one for Gitlab Pages
- 'if [ ! -f ".ci/_config_$CI_COMMIT_REF_NAME.yml" ]; then mv .ci/_config_gitlab_pages.yml .ci/_config_$CI_COMMIT_REF_NAME.yml; fi'
- |
if [ $CI_MERGE_REQUEST_ID ]; then
export CI_COMMIT_REF_NAME="develop";
else
echo "url: https://$CI_PROJECT_NAMESPACE.$CI_PAGES_DOMAIN" >> .ci/_config_gitlab_pages.yml;
echo "baseurl: /$CI_PROJECT_NAME" >> .ci/_config_gitlab_pages.yml;
# If there is no config for the current branch, use the one for Gitlab Pages
if [ ! -f ".ci/_config_$CI_COMMIT_REF_NAME.yml" ]; then
mv .ci/_config_gitlab_pages.yml .ci/_config_$CI_COMMIT_REF_NAME.yml;
fi
fi
# Display, which configuration is used
- 'echo "Configuration: " && cat ".ci/_config_$CI_COMMIT_REF_NAME.yml"'
- |
echo "Configuration: " && cat ".ci/_config_$CI_COMMIT_REF_NAME.yml"
# Run Jekyll with custom configuration
- bundle exec jekyll build -d build --config "_config.yml,.ci/_config_$CI_COMMIT_REF_NAME.yml"
# check
# ------------------------------------------------------------------------------------
check:links:
stage: check
image: $CI_REGISTRY/r3/apps/tailorbird/linkchecker
rules:
- if: $CI_COMMIT_REF_NAME
- if: $CI_MERGE_REQUEST_ID
allow_failure: true
before_script:
- cp .tmp/whitelist.txt build/.
- cp -r build /check
- cd /
script:
- python link_check.py
# generate
# ------------------------------------------------------------------------------------
......@@ -163,4 +189,4 @@ trigger:
tags:
- privileged
script:
- curl --silent --output /dev/null -X POST -F token=$INTERNAL_TRIGGER_TOKEN -F ref=$CI_COMMIT_BRANCH $INTERNAL_REPO
\ No newline at end of file
- curl --silent --output /dev/null -X POST -F token=$INTERNAL_TRIGGER_TOKEN -F ref=$CI_COMMIT_BRANCH $INTERNAL_REPO
......@@ -19,8 +19,8 @@ description: >- # this means to ignore newlines until "baseurl:"
This page is an index for lab cards that are intended to provide practical guidance in implementing Data Management, Data Protection and IT setup.
# URL settings (the most difficult part, please refer to the guide)
baseurl: "/" # the subpath of your site, e.g. /gitlab-repository-name
url: "https://localhost" # the base hostname & protocol for your site, e.g. http://gitlab-namespace-name.pages.uni.lu/
baseurl: "/howto-cards" # the subpath of your site, e.g. /gitlab-repository-name
url: "https://howto.lcsb.uni.lu" # the base hostname & protocol for your site, e.g. http://gitlab-namespace-name.pages.uni.lu/
# Banner settings
banner: howto-card # When you have custom images, change this setting's value to the name of the folder containing them
......
......@@ -31,7 +31,7 @@ In case you are not a member of the university, please ask your collaborator at
## (Re-)Activating LUMS account upon receiving credentials or after password reset
The password you will receive from the system administrators is temporary and **it will expire after one month**, so you have to reset it as soon as possible. It is usually sent as a link to [PrivateBin](https://howto.lcsb.uni.lu/?passwords). **This link is valid for one week and is deleted after you see it once**, thus you need to keep it open until you have followed these steps to reset your password:
The password you will receive from the system administrators is temporary and **it will expire after one month**, so you have to reset it as soon as possible. It is usually sent as a link to [PrivateBin]({{ '/?access:passwords' | relative_url }}). **This link is valid for one week and is deleted after you see it once**, thus you need to keep it open until you have followed these steps to reset your password:
1. Go to [lums.uni.lu](https://lums.uni.lu) and login with your LUMS username and the temporary password you received.<br>**NOTE:** If you experience any issues at this step, try to use *Chrome* browser for this procedure.<br/><img src="img/lums_login.png" height="200px"><br/>
Should the login look different from this, e.g. like a pop-up, click on **x** or **Cancel** to close it.
......@@ -43,7 +43,7 @@ The password you will receive from the system administrators is temporary and **
5. Enter a new password of your choice in the field **"New Password"** and repeat the same password in the field **"Verify Password"**. This is the actual password you need to use from now on, so make sure to remember it.
6. Click on **"Reset Password and Login"**.
It is a good practice to use a password manager which will not only help you to keep track of your passwords so you will never forget it, but it will also help you to generate passwords which are very safe (see [HowTo card](https://howto.lcsb.uni.lu/?passwords) on password management for more detail).
It is a good practice to use a password manager which will not only help you to keep track of your passwords so you will never forget it, but it will also help you to generate passwords which are very safe (see [HowTo card]({{ '/?access:passwords' | relative_url }}) on password management for more detail).
## Changing password for LUMS accounts
If you want to change your password for the LUMS account, you can:
......
......@@ -8,7 +8,7 @@ redirect_from:
---
# Git clients
Installation instructions are provided [here](https://howto.lcsb.uni.lu/?contribute:install-git).
Installation instructions are provided [here]({{ '/?contribute:install-git' | relative_url }}).
## GUI clients
......
......@@ -9,7 +9,7 @@ redirect_from:
# Installation of Git
Several git clients are presented [here](https://howto.lcsb.uni.lu/?contribute:git-clients).
Several git clients are presented [here]({{ '/?contribute:git-clients' | relative_url }}).
For most of the git clients, `git` is actually required to be installed on the system. You can find installers for Windows, MacOS and Linux on the [Git webpage](https://git-scm.com/downloads).
......@@ -17,4 +17,4 @@ If you are using Visual Studio Code, and if you want to contribute to a reposito
```bash
ssh git@gitlab.lcsb.uni.lu -p8022
```
Then, press Enter and type `yes`. This will add the git-r3lab server to the known hosts on your system.
\ No newline at end of file
Then, press Enter and type `yes`. This will add the Gitlab server to the known hosts on your system.
\ No newline at end of file
......@@ -132,4 +132,4 @@ Markdown file can be written in any text editor. There are many editors supporti
* ByWord
* Online
* [GitHub pages](https://pages.github.com/)
* [stackedit.io](stackedit.io)
* [stackedit.io](https://stackedit.io)
......@@ -59,7 +59,7 @@ For each how-to card, it is strongly recommended to create a new branch.
## Step 4: Write the procedure in Markdown
If you need more details on how to do this, refer to the [corresponding howto-card](.../write-markdown).
If you need more details on how to do this, refer to the [corresponding howto-card](/?contribute:markdown).
## Step 5: Commit your changes
......
......@@ -13,7 +13,10 @@ This shows how to use Gitlab Web IDE to add or edit a howto-card:
Before you start, please make sure that you have the correct access rights to edit the repository.
*Note: If you do not have the rights to edit the repository, you can either ask for permissions by contacting the repository maintainers, or use your own copy of the repository (called a `fork`). Please follow [these instructions](https://docs.gitlab.com/ee/user/project/repository/forking_workflow.html#creating-a-fork) to obtain your copy (fork). You can then [mirror the fork automatically](/cards/contribute:mirror-fork) to keep in sync.*
*Note: If you do not have the rights to edit the repository, you can either ask for permissions by contacting
the repository maintainers, or use your own copy of the repository (called a `fork`).
Please follow [these instructions](https://docs.gitlab.com/ee/user/project/repository/forking_workflow.html#creating-a-fork) to
obtain your copy (fork). You can then [mirror the fork automatically]({{ '/?contribute:mirror-fork' | relative_url }}) to keep in sync.*
A clear difference between editing a file in a git-tracked repository compared to a live document is that you can make an edit directly, and then suggesting the change. You can make this edit on your own version (called a `branch`), and then request to merge your changes into the main repository (through opening a `merge request`).
......
......@@ -85,7 +85,7 @@ You can use the official [IBM Aspera Diagnostic Tool](https://test-connect.asper
### **Using Microsoft Edge browser**
Microsoft Edge browser requires to download and install [IBM Aspera Connect for Edge](https://www.microsoft.com/store/productId/9N6XL57H8BMG).
Microsoft Edge browser requires to download and install [IBM Aspera Connect for Edge](https://microsoftedge.microsoft.com/addons/detail/ibm-aspera-connect/kbffkbiljjejklcpnfmoiaehplhcifki).
### **UDP/TCP port and firewall**
......
......@@ -44,7 +44,7 @@ When sharing research data, one should observe the following guidance using Ownc
## Installing Owncloud Desktop Client
**Important:** OwnCloud is fetching information from LUMS, therefore changes like a password reset (see How-to cards [Changing password for LUMS accounts](https://howto.lcsb.uni.lu/?lums-passwords)) should be performed **only** via LUMS!
**Important:** OwnCloud is fetching information from LUMS, therefore changes like a password reset (see How-to cards [Changing password for LUMS accounts]({{ '/?lums-passwords' | relative_url }}) should be performed **only** via LUMS!
1. Please try following link to download the owncloud desktop client depending on your operating system (this is an example for Windows, similar steps for other OS) -
......
......@@ -11,7 +11,7 @@ redirect_from:
In case you need to work remotely, there are several tools and services that you can use to
stay connected and continue your work.
An important reference is the [list of important links](https://howto.lcsb.uni.lu/?general:links).
An important reference is the [list of important links]({{ '/?general:links' | relative_url }}).
<p style="color:grey"><b>Note:</b> If the link does not work, please refer to the section <b>Connecting to the university network</b> since you are most probably not connected to the university internal network.</p>
......@@ -20,12 +20,12 @@ There is also a set of [tutorials](https://remote.uni.lu/) prepared by the unive
## Connecting to the university network
If you want to access the university infrastructure, such as the servers or storage, you can do so by connecting
via [Virtual Private Network (VPN)](http://vpn.uni.lu). All details are given [here](https://howto.lcsb.uni.lu/?access:vpn-cerbere-access).
via [Virtual Private Network (VPN)](http://vpn.uni.lu). All details are given [here]({{ '/?access:vpn-cerbere-access' | relative_url }}).
## Instant messaging
One of the recommended tools to communicate is Slack. Each research group can sign up for free and
create their own Slack instance on [slack.com](www.slack.com).
create their own Slack instance on [slack.com](https://www.slack.com).
## Video conferencing and virtual meetings
......@@ -43,7 +43,7 @@ In case you want to do video conferences, but do not have a webcam, you can use
For Android phones you can use [DroidCam Wireless Webcam](http://www.dev47apps.com/). It comes with a desktop client for Windows and Linux, but also works on MacOS without the client.
On Apple iPhones (and Android as well) you can use [EpocCam](http://www.kinoni.com/). This requires you to install drivers on your computer as well.
On Apple iPhones (and Android as well) you can use [EpocCam](https://www.elgato.com/en/epoccam). This requires you to install drivers on your computer as well.
Both apps support connecting to your computer either via WiFi or USB. EpocCam supports USB connection only for iOS, though. With DroidCam, you need to [install ADB, enable developer options and USB debugging](http://www.dev47apps.com/droidcam/connect/), if you want to connect your phone via USB.
......@@ -65,8 +65,8 @@ You can use the following tools to take notes or writing manuscripts together:
When on-the-go or working remotely, the [LCSB ownCloud](https://owncloud.lcsb.uni.lu) is the best way to share and work collaboratively on files.
More details on how to set up ownCloud are provided [here](https://howto.lcsb.uni.lu/?exchange-channels:owncloud). You can share your files and folders
directly with your collaborators by following [these instructions](https://howto.lcsb.uni.lu/?exchange-channels:owncloud:share-files).
More details on how to set up ownCloud are provided [here]({{ '/?exchange-channels:owncloud' | relative_url }}). You can share your files and folders
directly with your collaborators by following [these instructions]({{ '/?exchange-channels:owncloud:share-files' | relative_url }}).
## Development of code
......
......@@ -40,7 +40,7 @@ Download and install `7zip` from [here](https://www.7-zip.org/)
3. Enter a password and make a copy of the encryption password :key:
4. Click save to save the encrypted zip file
![](./media/7zip-encryption-windows.mp4?width=400)
![](media/7zip-encryption-windows.mp4?width=400)
## Calculate the MD5 checksum with `Simple Checksum Calculator`
......@@ -49,7 +49,7 @@ Download and install `7zip` from [here](https://www.7-zip.org/)
3. Click on the file icon box in the left panel and load the encrypted zip file
4. Make a copy of the MD5 checksum key :label:
![](./media/Get_MD5_checksum_windows.mp4)
![](media/Get_MD5_checksum_windows.mp4)
## Data upload
......
......@@ -9,7 +9,7 @@ redirect_from:
# Encrypting the Startup Disk for Your Laptop/Desktop
Encrypting an entire hard disk is an effective protective measure against computer theft and loss. In this lab card we provide instructions for switching on disk encryption on macOS and Windows platforms.
**IMPORTANT NOTICE:** One important requirement of using Encryption is that you must manage your Encryption Passwords/Keys. Failing to do so will mean **loosing your data indefinitely**. In this [link](https://howto.lcsb.uni.lu/?access:passwords) we list tools that can be used for password management. **Please make sure you have arrangements for password management before starting the encryption of data**.
**IMPORTANT NOTICE:** One important requirement of using Encryption is that you must manage your Encryption Passwords/Keys. Failing to do so will mean **loosing your data indefinitely**. In this [link]({{ '/?access:passwords' | relative_url }}) we list tools that can be used for password management. **Please make sure you have arrangements for password management before starting the encryption of data**.
## macOS
......
......@@ -9,7 +9,7 @@ redirect_from:
# Encrypting Files and Folders
Encryption is an effective measure to protect sensitive data. In this lab card we provide instructions for file/folder encryption on platforms commonly used by LCSB staff.
**IMPORTANT NOTICE:** One important requirement of using Encryption is that you must manage your Encryption Passwords/Keys. Failing to do so will mean **loosing your data indefinitely**. In this [link](https://howto.lcsb.uni.lu/?access:passwords) we list tools that can be used for password management. **Please make sure you have arrangements for password management before starting the encryption of data**.
**IMPORTANT NOTICE:** One important requirement of using Encryption is that you must manage your Encryption Passwords/Keys. Failing to do so will mean **loosing your data indefinitely**. In this [link]({{ '/?access:passwords' | relative_url }}) we list tools that can be used for password management. **Please make sure you have arrangements for password management before starting the encryption of data**.
## macOS
The built-in mechanism for file-level encryption on a Mac is Encrypted Disk Images (*.dmg* files). In order to create a disk image:
......
......@@ -16,7 +16,7 @@ Good file name follows three basic principles:
* human readable
* plays well with default ordering
If you are looking for information on how to organize and structure your folders, you may find this [dedicated card](https://howto.lcsb.uni.lu/?integrity:organization) helpful.
If you are looking for information on how to organize and structure your folders, you may find this [dedicated card]({{ '/?integrity:organization' | relative_url }}) helpful.
## Machine readable
......@@ -56,7 +56,7 @@ Beware of typos and avoid using multiple names varying in small ways unless it h
## Exploiting machine readable names
You may already have a lot of files collected for your project or you have received big dataset from one of your collaborators. Then you might think about organizing and renaming them to be compliant with your new or existing naming policy.
If the names are consistent and you don't want to loose time renaming them by hand, you may try to use dedicated tools (e.g. [PSRenamer](https://www.powersurgepub.com/products/psrenamer/index.html)) or simple commands in your command line (**rename** for Mac and Linux, **ren** for Windows).
If the names are consistent and you don't want to loose time renaming them by hand, you may try to use dedicated tools (e.g. [PSRenamer](https://github.com/hbowie/psrenamer) or simple commands in your command line (**rename** for Mac and Linux, **ren** for Windows).
Once your skills develop, you will be able to use machines and machine readable file names to perform advanced operations on them, e.g. search using regular expression.
Imagine folder with thousands of files. Running simple R command
......
......@@ -21,7 +21,7 @@ able to find back the files easily.
A key rule is to name the folders with straightforward but explicit names, and to include the decision tree of where to put a given file inside the folder structure. In fact, the directory tree should in some sense
reflect the decision tree.
More details on naming files and folders are provided in the dedicated [file naming card](https://howto.lcsb.uni.lu/?integrity:naming).
More details on naming files and folders are provided in the dedicated [file naming card]({{ '/?integrity:naming' | relative_url }}).
## Suggestions
......
......@@ -42,7 +42,7 @@ For reproducibility purposes, the collected data should be always exported from
### Table
- Keep header column names machine readable. You can follow the same best practices as for file naming (see our [Card on file naming](../naming/file_naming.md)).
- Keep header column names machine readable. You can follow the same best practices as for file naming (see our [Card on file naming]({{ '/?integrity:naming' | relative_url }})).
- Keep values in columns atomic.
- Use primary keys - values in one particular column should be unique for the whole table. This will allow you to create unique references pointing to one and only one observation/record.
- Do not insert empty rows or columns which would split the table in two.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment