Git clone ou git pull

Dump des projets git plus malin

src/resifdatadump

@@ -278,38 +278,48 @@ iput -f $LOCAL_REPORT $IRODS_REPORT
 if [ -z ${GITLAB_TOKEN+x} ]; then
     echo "GITLAB_TOKEN is not set. We will NOT dump gitlab projects"
 else
-    username="ccdump"
-    # number of results per pages (max = 100)
-    per_page=50
-    url_repositories="https://gricad-gitlab.univ-grenoble-alpes.fr/api/v4/groups/192/projects?simple=true&include_subgroups=true&per_page=$per_page"
-    # total number of projects
-    total=$(curl -i -s --header "PRIVATE-TOKEN: $GITLAB_TOKEN" "$url_repositories" | awk '/x-total:/ {printf "%.0f\n", $2}')
-    # check if $total is a number:
-    re_is_num='^[0-9]+$'
-    if [[ "$total" =~ $re_is_num ]]; then
-        # max page calculation
-        max_page=$(($total / $per_page +1))
-        for page in `seq $max_page`; do
-            # get projects list
-            project_list=$(curl -s --header "PRIVATE-TOKEN: $GITLAB_TOKEN" $url_repositories"&page="$page | grep -o "\"http_url_to_repo\":[^ ,]\+" | xargs -n 1|sed -e 's/http_url_to_repo:https:\/\///')
-            for url in ${project_list[@]}; do
-                # This pattern captures the project name (group 2) and the possible subdirectories (group 1).
-                pattern="^.*\/OSUG\/RESIF\/(.*\/)*(.+.git)$"
-                if [[ "$url" =~ $pattern ]]; then
-                    destdir=$RESIFDD_WORKDIR/projects/${BASH_REMATCH[1]}
-                    project=${BASH_REMATCH[2]}
-                    echo "Cloning $url in $destdir/$project"
-                    mkdir -p $destdir
-                    git clone -q --bare https://$username:$GITLAB_TOKEN@$url $destdir/$project
-                fi
-            done
-        done
-        # Now compress all the project and send to irods
-        KEY=git_projects
-        tar cJf $RESIFDD_WORKDIR/${KEY}.tar.xz $RESIFDD_WORKDIR/projects
-        iput --retries 5 -T -f -X $RESIFDD_WORKDIR/${KEY}.restart $RESIFDD_WORKDIR/${KEY}.tar.xz .
+    KEY=git_projects
+    if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOVERY_FILE ; then
+        format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
     else
-        echo "ERROR: gitlab projects could not be dumped, maybe token or gitlab url is wrong ?"
+        username="ccdump"
+        # number of results per pages (max = 100)
+        per_page=50
+        url_repositories="https://gricad-gitlab.univ-grenoble-alpes.fr/api/v4/groups/192/projects?simple=true&include_subgroups=true&per_page=$per_page"
+        # total number of projects
+        total=$(curl -i -s --header "PRIVATE-TOKEN: $GITLAB_TOKEN" "$url_repositories" | awk '/x-total:/ {printf "%.0f\n", $2}')
+        # check if $total is a number:
+        re_is_num='^[0-9]+$'
+        if [[ "$total" =~ $re_is_num ]]; then
+            # max page calculation
+            max_page=$(($total / $per_page +1))
+            for page in `seq $max_page`; do
+                # get projects list
+                project_list=$(curl -s --header "PRIVATE-TOKEN: $GITLAB_TOKEN" $url_repositories"&page="$page | grep -o "\"http_url_to_repo\":[^ ,]\+" | xargs -n 1|sed -e 's/http_url_to_repo:https:\/\///')
+                for url in ${project_list[@]}; do
+                    # This pattern captures the project name (group 2) and the possible subdirectories (group 1).
+                    pattern="^.*\/OSUG\/RESIF\/(.*\/)*(.+.git)$"
+                    if [[ "$url" =~ $pattern ]]; then
+                        destdir=$RESIFDD_WORKDIR/projects/${BASH_REMATCH[1]}
+                        project=${BASH_REMATCH[2]}
+                        mkdir -p $destdir
+                        if [[ -d $destdir/$project ]]; then
+                            echo "Pulling $url in $destdir/$project"
+                            git -C $destdir/$project pull
+                        else
+                            echo "Cloning $url in $destdir/$project"
+                            git clone -q --bare https://$username:$GITLAB_TOKEN@$url $destdir/$project
+                        fi
+                    fi
+                done
+            done
+            # Now compress all the project and send to irods
+            tar cJf $RESIFDD_WORKDIR/${KEY}.tar.xz -C $RESIFDD_WORKDIR $RESIFDD_WORKDIR/projects
+            iput --retries 5 -T -f -X $RESIFDD_WORKDIR/${KEY}.restart $RESIFDD_WORKDIR/${KEY}.tar.xz .
+            rm -rf  $RESIFDD_WORKDIR/projects $RESIFDD_WORKDIR/${KEY}.tar.xz
+        else
+            echo "ERROR: gitlab projects could not be dumped, maybe token or gitlab url is wrong ?"
+        fi
     fi
 fi