bapung/gitea-runner-operator
1
0
Fork 0
mirror of https://github.com/bapung/gitea-runner-operator.git synced 2026-06-21 23:48:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement working reconciliation logic and documentation

Browse Source 
initial commit for working reconciliation logic, no automated test only manually tested for now
This commit is contained in:
Bagas Purwa S
2026-01-12 22:57:22 +08:00
committed by GitHub
parent b638d72402
commit 86e92c5e72
18 changed files with 810 additions and 655 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
internal/controller/runnergroup_controller.go
View File

@@ -21,6 +21,7 @@ import (
"fmt"
"math/rand"
"strings"
"sync"
"time"
batchv1 "k8s.io/api/batch/v1"
@@ -40,8 +41,9 @@ import (
// RunnerGroupReconciler reconciles a RunnerGroup object
type RunnerGroupReconciler struct {
client.Client
Scheme *runtime.Scheme
GiteaClient gitea.Client
Scheme *runtime.Scheme
GiteaClient gitea.Client
SpawnedJobsCache sync.Map
}
// +kubebuilder:rbac:groups=gitea.bpg.pw,resources=runnergroups,verbs=get;list;watch;create;update;patch;delete
@@ -117,56 +119,93 @@ func (r *RunnerGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request)
logger.Info("Checking Gitea for queued jobs", "url", runnerGroup.Spec.GiteaURL, "scope", runnerGroup.Spec.Scope)
// Calculate effective labels (spec labels + defaults)
effectiveLabels := r.getEffectiveLabels(runnerGroup.Spec.Labels)
// Query for queued workflow runs
queuedJobs, err := r.GiteaClient.GetQueuedRuns(
stats, err := r.GiteaClient.GetRunnerStats(
ctx,
runnerGroup.Spec.GiteaURL,
authToken,
runnerGroup.Spec.Scope,
runnerGroup.Spec.Org,
runnerGroup.Spec.User,
runnerGroup.Spec.Repo,
runnerGroup.Spec.Labels,
effectiveLabels,
)
if err != nil {
logger.Error(err, "Failed to query Gitea for queued runs")
logger.Error(err, "Failed to query Gitea for runner stats")
return ctrl.Result{RequeueAfter: 10 * time.Second}, err
}
logger.Info("Gitea query result", "queuedJobs", queuedJobs)
logger.Info("Gitea query result", "queuedJobs", len(stats.QueuedJobs))
// 6. Scale Up
// 6. Scale Up and Cache Management
availableSlots := runnerGroup.Spec.MaxActiveRunners - activeRunners
toSpawn := min(queuedJobs, availableSlots)
if toSpawn > 0 {
logger.Info("Spawning runners",
"queuedJobs", queuedJobs,
"availableSlots", availableSlots,
"toSpawn", toSpawn)
// Track current queued IDs for cache cleanup
currentQueuedIDs := make(map[int64]bool)
// Retrieve Registration Token from Secret
registrationToken, err := r.getSecretValue(ctx, runnerGroup.Namespace, runnerGroup.Spec.RegistrationTokenRef)
// Retrieve Registration Token from Secret (only if we need to spawn)
var registrationToken string
tokenFetched := false
for _, giteaJob := range stats.QueuedJobs {
currentQueuedIDs[giteaJob.ID] = true
if availableSlots <= 0 {
continue
}
// Check if we already spawned a runner for this job
if value, loaded := r.SpawnedJobsCache.Load(giteaJob.ID); loaded {
spawnTime := value.(time.Time)
if time.Since(spawnTime) < 5*time.Minute {
// Already handling this job recently
continue
}
// TTL expired (runner likely failed to start), retry spawning
logger.Info("Job stuck in queue for too long, retrying runner spawn", "giteaJobID", giteaJob.ID)
}
// Need to spawn a runner
if !tokenFetched {
registrationToken, err = r.getSecretValue(ctx, runnerGroup.Namespace, runnerGroup.Spec.RegistrationTokenRef)
if err != nil {
logger.Error(err, "Failed to get registration token from secret")
return ctrl.Result{}, err
}
tokenFetched = true
}
job, err := r.constructJobForRunnerGroup(runnerGroup, registrationToken, effectiveLabels)
if err != nil {
logger.Error(err, "Failed to get registration token from secret")
logger.Error(err, "Failed to construct Job")
return ctrl.Result{}, err
}
// Spawn jobs
for i := 0; i < toSpawn; i++ {
job, err := r.constructJobForRunnerGroup(runnerGroup, registrationToken)
if err != nil {
logger.Error(err, "Failed to construct Job")
return ctrl.Result{}, err
}
if err := r.Create(ctx, job); err != nil {
logger.Error(err, "Failed to create Job", "jobName", job.Name)
return ctrl.Result{}, err
}
logger.Info("Created Job", "jobName", job.Name)
if err := r.Create(ctx, job); err != nil {
logger.Error(err, "Failed to create Job", "jobName", job.Name)
return ctrl.Result{}, err
}
logger.Info("Created Job for Gitea Run", "jobName", job.Name, "giteaJobID", giteaJob.ID)
// Mark as spawned
r.SpawnedJobsCache.Store(giteaJob.ID, time.Now())
availableSlots--
}
// Cleanup cache: remove jobs that are no longer queued in Gitea
r.SpawnedJobsCache.Range(func(key, value any) bool {
jobID := key.(int64)
if !currentQueuedIDs[jobID] {
// Job is no longer in the queue (running, completed, or cancelled)
r.SpawnedJobsCache.Delete(key)
}
return true
})
// 7. Requeue for continuous polling
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
}
@@ -191,8 +230,43 @@ func (r *RunnerGroupReconciler) getSecretValue(ctx context.Context, namespace st
return string(value), nil
}
// getEffectiveLabels merges spec labels with default labels
func (r *RunnerGroupReconciler) getEffectiveLabels(specLabels []string) []string {
defaultLabels := []string{
"ubuntu-latest:docker://node:16-bullseye",
"ubuntu-22.04:docker://node:16-bullseye",
"ubuntu-20.04:docker://node:16-bullseye",
"ubuntu-18.04:docker://node:16-buster",
}
effectiveLabels := make([]string, len(specLabels))
copy(effectiveLabels, specLabels)
for _, defaultLabel := range defaultLabels {
// Check if this default label key is already overridden in specLabels
// defaultLabel format is "key:schema"
parts := strings.SplitN(defaultLabel, ":", 2)
key := parts[0]
found := false
for _, specLabel := range specLabels {
// Spec label can be "key" or "key:schema"
if specLabel == key || strings.HasPrefix(specLabel, key+":") {
found = true
break
}
}
if !found {
effectiveLabels = append(effectiveLabels, defaultLabel)
}
}
return effectiveLabels
}
// constructJobForRunnerGroup creates a Job object for the RunnerGroup
func (r *RunnerGroupReconciler) constructJobForRunnerGroup(runnerGroup *giteav1alpha1.RunnerGroup, registrationToken string) (*batchv1.Job, error) {
func (r *RunnerGroupReconciler) constructJobForRunnerGroup(runnerGroup *giteav1alpha1.RunnerGroup, registrationToken string, labels []string) (*batchv1.Job, error) {
// Generate random suffix for name
name := fmt.Sprintf("%s-%s", runnerGroup.Name, randString(8))
@@ -201,13 +275,14 @@ func (r *RunnerGroupReconciler) constructJobForRunnerGroup(runnerGroup *giteav1a
{Name: "GITEA_INSTANCE_URL", Value: runnerGroup.Spec.GiteaURL},
{Name: "GITEA_RUNNER_REGISTRATION_TOKEN", Value: registrationToken},
{Name: "GITEA_RUNNER_EPHEMERAL", Value: "true"},
{Name: "GITEA_RUNNER_NAME", Value: name},
{Name: "DOCKER_HOST", Value: "tcp://localhost:2376"},
{Name: "DOCKER_CERT_PATH", Value: "/certs/client"},
{Name: "DOCKER_TLS_VERIFY", Value: "1"},
}
if len(runnerGroup.Spec.Labels) > 0 {
labelsStr := strings.Join(runnerGroup.Spec.Labels, ",")
if len(labels) > 0 {
labelsStr := strings.Join(labels, ",")
envVars = append(envVars, corev1.EnvVar{Name: "GITEA_RUNNER_LABELS", Value: labelsStr})
}
@@ -276,14 +351,6 @@ func randString(length int) string {
return string(b)
}
// min returns the minimum of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}
// SetupWithManager sets up the controller with the Manager.
func (r *RunnerGroupReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).

42
internal/controller/runnergroup_controller_test.go
View File

@@ -25,11 +25,19 @@ import (
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
giteav1alpha1 "github.com/bapung/gitea-runner-operator/api/v1alpha1"
"github.com/bapung/gitea-runner-operator/internal/gitea"
)
type fakeGiteaClient struct{}
func (c *fakeGiteaClient) GetRunnerStats(ctx context.Context, giteaURL, authToken string, scope giteav1alpha1.RunnerGroupScope, org string, user string, repo string, labels []string) (*gitea.RunnerStats, error) {
return &gitea.RunnerStats{QueuedJobs: []gitea.ActionWorkflowJob{}}, nil
}
var _ = Describe("RunnerGroup Controller", func() {
Context("When reconciling a resource", func() {
const resourceName = "test-resource"
@@ -43,6 +51,21 @@ var _ = Describe("RunnerGroup Controller", func() {
runnergroup := &giteav1alpha1.RunnerGroup{}
BeforeEach(func() {
By("creating the secret")
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "gitea-secret",
Namespace: "default",
},
Data: map[string][]byte{
"token": []byte("dummy"),
"auth": []byte("dummy"),
},
}
if err := k8sClient.Create(ctx, secret); err != nil && !errors.IsAlreadyExists(err) {
Expect(err).To(Succeed())
}
By("creating the custom resource for the Kind RunnerGroup")
err := k8sClient.Get(ctx, typeNamespacedName, runnergroup)
if err != nil && errors.IsNotFound(err) {
@@ -51,7 +74,19 @@ var _ = Describe("RunnerGroup Controller", func() {
Name: resourceName,
Namespace: "default",
},
// TODO(user): Specify other spec details if needed.
Spec: giteav1alpha1.RunnerGroupSpec{
Scope: giteav1alpha1.RunnerGroupScopeGlobal,
GiteaURL: "https://gitea.example.com",
MaxActiveRunners: 1,
RegistrationTokenRef: corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "gitea-secret"},
Key: "token",
},
AuthTokenRef: corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "gitea-secret"},
Key: "auth",
},
},
}
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
}
@@ -69,8 +104,9 @@ var _ = Describe("RunnerGroup Controller", func() {
It("should successfully reconcile the resource", func() {
By("Reconciling the created resource")
controllerReconciler := &RunnerGroupReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
Client: k8sClient,
Scheme: k8sClient.Scheme(),
GiteaClient: &fakeGiteaClient{},
}
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{

408
internal/gitea/client.go
View File

@@ -31,17 +31,22 @@ import (
// Client defines the interface for interacting with Gitea API
type Client interface {
// GetQueuedRuns queries Gitea for queued workflow runs matching the scope and labels
// Returns the count of queued jobs that match the criteria
GetQueuedRuns(
// GetRunnerStats queries Gitea for queued workflow runs matching the scope and labels
GetRunnerStats(
ctx context.Context,
giteaURL string,
authToken string,
scope v1alpha1.RunnerGroupScope,
org string,
user string,
repo string,
labels []string,
) (int, error)
) (*RunnerStats, error)
}
// RunnerStats contains lists of jobs in different states
type RunnerStats struct {
QueuedJobs []ActionWorkflowJob
}
// HTTPClient is the default implementation of the Gitea Client interface
@@ -107,153 +112,163 @@ type ActionWorkflowJob struct {
RunnerName string `json:"runner_name"`
}
// GetQueuedRuns implements the Client interface
func (c *HTTPClient) GetQueuedRuns(
// GetRunnerStats implements the Client interface
func (c *HTTPClient) GetRunnerStats(
ctx context.Context,
giteaURL string,
authToken string,
scope v1alpha1.RunnerGroupScope,
org string,
user string,
repo string,
labels []string,
) (int, error) {
) (*RunnerStats, error) {
switch scope {
case v1alpha1.RunnerGroupScopeRepo:
return c.getQueuedRunsForRepo(ctx, giteaURL, authToken, org, repo, labels)
return c.getRunnerStatsForRepo(ctx, giteaURL, authToken, org, repo, labels)
case v1alpha1.RunnerGroupScopeOrg:
return c.getQueuedRunsForOrg(ctx, giteaURL, authToken, org, labels)
return c.getRunnerStatsForOrg(ctx, giteaURL, authToken, org, labels)
case v1alpha1.RunnerGroupScopeUser:
return c.getRunnerStatsForUser(ctx, giteaURL, authToken, user, labels)
case v1alpha1.RunnerGroupScopeGlobal:
return c.getQueuedRunsGlobal(ctx, giteaURL, authToken, labels)
return c.getRunnerStatsGlobal(ctx, giteaURL, authToken, labels)
default:
return 0, fmt.Errorf("unknown scope: %s", scope)
return nil, fmt.Errorf("unknown scope: %s", scope)
}
}
// getQueuedRunsForRepo fetches queued runs for a specific repository
func (c *HTTPClient) getQueuedRunsForRepo(ctx context.Context, giteaURL, authToken, owner, repo string, labels []string) (int, error) {
// Use jobs endpoint since it contains the runner labels we need for filtering
// getRunnerStatsForRepo fetches queued runs for a specific repository
func (c *HTTPClient) getRunnerStatsForRepo(ctx context.Context, giteaURL, authToken, owner, repo string, labels []string) (*RunnerStats, error) {
endpoint := fmt.Sprintf("%s/api/v1/repos/%s/%s/actions/jobs", strings.TrimSuffix(giteaURL, "/"), owner, repo)
return c.fetchWorkflowJobs(ctx, endpoint, authToken, labels)
return c.fetchRunnerStats(ctx, endpoint, authToken, labels)
}
// getQueuedRunsForOrg fetches queued runs for all repos under an organization
func (c *HTTPClient) getQueuedRunsForOrg(ctx context.Context, giteaURL, authToken, org string, labels []string) (int, error) {
// Use direct org-level jobs endpoint for better performance
// getRunnerStatsForOrg fetches queued runs for all repos under an organization
func (c *HTTPClient) getRunnerStatsForOrg(ctx context.Context, giteaURL, authToken, org string, labels []string) (*RunnerStats, error) {
endpoint := fmt.Sprintf("%s/api/v1/orgs/%s/actions/jobs", strings.TrimSuffix(giteaURL, "/"), org)
return c.fetchWorkflowJobs(ctx, endpoint, authToken, labels)
return c.fetchRunnerStats(ctx, endpoint, authToken, labels)
}
// getQueuedRunsGlobal fetches queued runs using admin-level API for global scope
func (c *HTTPClient) getQueuedRunsGlobal(ctx context.Context, giteaURL, authToken string, labels []string) (int, error) {
// Use admin-level jobs endpoint which provides global view of all queued jobs
// getRunnerStatsForUser fetches queued runs for all repos owned by a user
func (c *HTTPClient) getRunnerStatsForUser(ctx context.Context, giteaURL, authToken, user string, labels []string) (*RunnerStats, error) {
repos, err := c.fetchReposForUser(ctx, giteaURL, authToken, user)
if err != nil {
return nil, err
}
var allQueuedJobs []ActionWorkflowJob
for _, repo := range repos {
endpoint := fmt.Sprintf("%s/api/v1/repos/%s/%s/actions/jobs", strings.TrimSuffix(giteaURL, "/"), repo.Owner.Login, repo.Name)
stats, err := c.fetchRunnerStats(ctx, endpoint, authToken, labels)
if err != nil {
return nil, err
}
allQueuedJobs = append(allQueuedJobs, stats.QueuedJobs...)
}
return &RunnerStats{
QueuedJobs: allQueuedJobs,
}, nil
}
// getRunnerStatsGlobal fetches queued runs using admin-level API for global scope
func (c *HTTPClient) getRunnerStatsGlobal(ctx context.Context, giteaURL, authToken string, labels []string) (*RunnerStats, error) {
endpoint := fmt.Sprintf("%s/api/v1/admin/actions/jobs", strings.TrimSuffix(giteaURL, "/"))
return c.fetchWorkflowJobs(ctx, endpoint, authToken, labels)
return c.fetchRunnerStats(ctx, endpoint, authToken, labels)
}
func (c *HTTPClient) fetchRunnerStats(ctx context.Context, endpoint, authToken string, labels []string) (*RunnerStats, error) {
queuedJobs, err := c.fetchWorkflowJobs(ctx, endpoint, authToken, labels, []string{"queued", "waiting", "pending"})
if err != nil {
return nil, err
}
return &RunnerStats{
QueuedJobs: queuedJobs,
}, nil
}
// fetchWorkflowJobs fetches workflow jobs from a given endpoint with label filtering and pagination
func (c *HTTPClient) fetchWorkflowJobs(ctx context.Context, endpoint, authToken string, labels []string) (int, error) {
totalCount := 0
page := 1
limit := 50 // Default page size
func (c *HTTPClient) fetchWorkflowJobs(ctx context.Context, endpoint, authToken string, labels []string, statuses []string) ([]ActionWorkflowJob, error) {
var allJobs []ActionWorkflowJob
for {
u, err := url.Parse(endpoint)
if err != nil {
return 0, err
}
q := u.Query()
q.Set("status", "queued")
q.Set("page", fmt.Sprintf("%d", page))
q.Set("limit", fmt.Sprintf("%d", limit))
u.RawQuery = q.Encode()
for _, status := range statuses {
page := 1
limit := 50 // Default page size
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil {
return 0, err
}
for {
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
q := u.Query()
q.Set("status", status)
q.Set("page", fmt.Sprintf("%d", page))
q.Set("limit", fmt.Sprintf("%d", limit))
u.RawQuery = q.Encode()
req.Header.Set("Authorization", "token "+authToken)
req.Header.Set("Accept", "application/json")
fmt.Printf("DEBUG: Fetching jobs from %s\n", u.String())
resp, err := c.httpClient.Do(req)
if err != nil {
return 0, err
}
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "token "+authToken)
req.Header.Set("Accept", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
fmt.Printf("DEBUG: Request failed: %v\n", err)
return nil, err
}
fmt.Printf("DEBUG: Response status: %s\n", resp.Status)
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
fmt.Printf("DEBUG: Error body: %s\n", string(body))
return nil, c.handleHTTPError(resp.StatusCode, body, "fetch workflow jobs")
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return 0, c.handleHTTPError(resp.StatusCode, body, "fetch workflow jobs")
_ = resp.Body.Close()
fmt.Printf("DEBUG: Response body: %s\n", string(body))
var result ActionWorkflowJobsResponse
if err := json.Unmarshal(body, &result); err != nil {
fmt.Printf("DEBUG: Failed to decode response: %v\n", err)
return nil, err
}
fmt.Printf("DEBUG: Found %d jobs, total in Gitea: %d\n", len(result.Jobs), result.TotalCount)
// Filter and collect matching jobs for this page
matchedJobs := c.filterQueuedJobs(result.Jobs, labels)
fmt.Printf("DEBUG: %d jobs matched labels %v\n", len(matchedJobs), labels)
allJobs = append(allJobs, matchedJobs...)
// Break if we've fetched all available results
if len(result.Jobs) < limit {
break
}
page++
}
var result ActionWorkflowJobsResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
resp.Body.Close()
return 0, err
}
resp.Body.Close()
// Filter and count matching jobs for this page
pageCount := c.filterQueuedJobs(result.Jobs, labels)
totalCount += pageCount
// Break if we've fetched all available results
if len(result.Jobs) < limit {
break
}
page++
}
return totalCount, nil
return allJobs, nil
}
// fetchWorkflowRuns fetches workflow runs from a given endpoint (deprecated - use jobs for label filtering)
func (c *HTTPClient) fetchWorkflowRuns(ctx context.Context, endpoint, authToken string) ([]ActionWorkflowRun, error) {
// Add status=queued query parameter
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
q := u.Query()
q.Set("status", "queued")
u.RawQuery = q.Encode()
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "token "+authToken)
req.Header.Set("Accept", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, c.handleHTTPError(resp.StatusCode, body, "fetch workflow runs")
}
var result ActionWorkflowRunsResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
return result.WorkflowRuns, nil
}
// fetchOrgRepos fetches all repositories under an organization with pagination
func (c *HTTPClient) fetchOrgRepos(ctx context.Context, giteaURL, authToken, org string) ([]Repository, error) {
// fetchReposForUser fetches all repositories owned by a specific user with pagination
func (c *HTTPClient) fetchReposForUser(ctx context.Context, giteaURL, authToken, username string) ([]Repository, error) {
var allRepos []Repository
page := 1
limit := 50
for {
endpoint := fmt.Sprintf("%s/api/v1/orgs/%s/repos", strings.TrimSuffix(giteaURL, "/"), org)
endpoint := fmt.Sprintf("%s/api/v1/users/%s/repos", strings.TrimSuffix(giteaURL, "/"), username)
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
@@ -263,6 +278,8 @@ func (c *HTTPClient) fetchOrgRepos(ctx context.Context, giteaURL, authToken, org
q.Set("limit", fmt.Sprintf("%d", limit))
u.RawQuery = q.Encode()
fmt.Printf("DEBUG: Fetching repos for user %s from %s\n", username, u.String())
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil {
return nil, err
@@ -273,131 +290,28 @@ func (c *HTTPClient) fetchOrgRepos(ctx context.Context, giteaURL, authToken, org
resp, err := c.httpClient.Do(req)
if err != nil {
fmt.Printf("DEBUG: Request failed: %v\n", err)
return nil, err
}
fmt.Printf("DEBUG: Response status: %s\n", resp.Status)
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
_ = resp.Body.Close()
fmt.Printf("DEBUG: Error body: %s\n", string(body))
return nil, c.handleHTTPError(resp.StatusCode, body, "fetch user repos")
}
var repos []Repository
if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil {
resp.Body.Close()
return nil, err
}
resp.Body.Close()
allRepos = append(allRepos, repos...)
if len(repos) < limit {
break
}
page++
}
return allRepos, nil
}
// fetchAllOrgs fetches all organizations visible to the authenticated user with pagination
func (c *HTTPClient) fetchAllOrgs(ctx context.Context, giteaURL, authToken string) ([]Organization, error) {
var allOrgs []Organization
page := 1
limit := 50
for {
endpoint := fmt.Sprintf("%s/api/v1/user/orgs", strings.TrimSuffix(giteaURL, "/"))
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
q := u.Query()
q.Set("page", fmt.Sprintf("%d", page))
q.Set("limit", fmt.Sprintf("%d", limit))
u.RawQuery = q.Encode()
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "token "+authToken)
req.Header.Set("Accept", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return nil, c.handleHTTPError(resp.StatusCode, body, "fetch org repos")
}
var orgs []Organization
if err := json.NewDecoder(resp.Body).Decode(&orgs); err != nil {
resp.Body.Close()
return nil, err
}
resp.Body.Close()
allOrgs = append(allOrgs, orgs...)
if len(orgs) < limit {
break
}
page++
}
return allOrgs, nil
}
// fetchUserRepos fetches all repositories owned by the authenticated user with pagination
func (c *HTTPClient) fetchUserRepos(ctx context.Context, giteaURL, authToken string) ([]Repository, error) {
var allRepos []Repository
page := 1
limit := 50
for {
endpoint := fmt.Sprintf("%s/api/v1/user/repos", strings.TrimSuffix(giteaURL, "/"))
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
q := u.Query()
q.Set("page", fmt.Sprintf("%d", page))
q.Set("limit", fmt.Sprintf("%d", limit))
u.RawQuery = q.Encode()
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "token "+authToken)
req.Header.Set("Accept", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return nil, c.handleHTTPError(resp.StatusCode, body, "fetch user orgs")
}
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
// fmt.Printf("DEBUG: Response body: %s\n", string(body))
var repos []Repository
if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil {
resp.Body.Close()
if err := json.Unmarshal(body, &repos); err != nil {
fmt.Printf("DEBUG: Failed to decode response: %v\n", err)
return nil, err
}
resp.Body.Close()
allRepos = append(allRepos, repos...)
@@ -412,44 +326,42 @@ func (c *HTTPClient) fetchUserRepos(ctx context.Context, giteaURL, authToken str
}
// filterQueuedJobs filters workflow jobs by labels
func (c *HTTPClient) filterQueuedJobs(jobs []ActionWorkflowJob, requiredLabels []string) int {
if len(requiredLabels) == 0 {
// No label filtering required, return all queued jobs
return len(jobs)
}
count := 0
func (c *HTTPClient) filterQueuedJobs(jobs []ActionWorkflowJob, runnerLabels []string) []ActionWorkflowJob {
var matched []ActionWorkflowJob
for _, job := range jobs {
if c.jobMatchesLabels(job.Labels, requiredLabels) {
count++
match := c.jobMatchesLabels(job.Labels, runnerLabels)
fmt.Printf("DEBUG: Job %d (Status: %s, Labels: %v) matches runner capabilities %v? %v\n", job.ID, job.Status, job.Labels, runnerLabels, match)
if match {
matched = append(matched, job)
}
}
return count
return matched
}
// jobMatchesLabels checks if a job's labels match the required labels
func (c *HTTPClient) jobMatchesLabels(jobLabels, requiredLabels []string) bool {
// Convert job labels to map for faster lookup
labelSet := make(map[string]bool)
for _, label := range jobLabels {
labelSet[label] = true
// jobMatchesLabels checks if a job's requirements are satisfied by the runner's supported labels
func (c *HTTPClient) jobMatchesLabels(jobLabels, supportedLabels []string) bool {
if len(jobLabels) == 0 {
return true
}
// Check if all required labels are present
for _, required := range requiredLabels {
if !labelSet[required] {
// For each label required by the job, check if the runner supports it
for _, req := range jobLabels {
found := false
for _, supp := range supportedLabels {
// Check for exact match or schema match (label:schema)
// e.g. Job asks for "ubuntu-latest", Runner has "ubuntu-latest:docker://..."
if req == supp || strings.HasPrefix(supp, req+":") {
found = true
break
}
}
if !found {
return false
}
}
return true
}
// filterQueuedRuns filters workflow runs by labels (deprecated - use filterQueuedJobs)
func (c *HTTPClient) filterQueuedRuns(runs []ActionWorkflowRun, labels []string) int {
// Legacy method - jobs should be used for label filtering
return len(runs)
}
// handleHTTPError provides specific error handling for different HTTP status codes
func (c *HTTPClient) handleHTTPError(statusCode int, body []byte, operation string) error {
switch statusCode {

195
internal/gitea/client_test.go
View File

@@ -27,16 +27,17 @@ import (
"github.com/bapung/gitea-runner-operator/api/v1alpha1"
)
func TestHTTPClient_GetQueuedRuns(t *testing.T) {
func TestHTTPClient_GetRunnerStats(t *testing.T) {
tests := []struct {
name string
scope v1alpha1.RunnerGroupScope
org string
repo string
labels []string
mockResponse ActionWorkflowJobsResponse
expectedCount int
expectedError bool
name string
scope v1alpha1.RunnerGroupScope
org string
user string
repo string
labels []string
mockResponse ActionWorkflowJobsResponse
expectedQueued int
expectedError bool
}{
{
name: "repo scope with matching labels",
@@ -51,38 +52,55 @@ func TestHTTPClient_GetQueuedRuns(t *testing.T) {
{ID: 2, Status: "queued", Labels: []string{"linux", "arm64"}},
},
},
expectedCount: 1,
expectedError: false,
expectedQueued: 1, // Job 1 matches
expectedError: false,
},
{
name: "org scope no label filtering",
name: "org scope no label filtering (matches all)",
scope: v1alpha1.RunnerGroupScopeOrg,
org: "testorg",
labels: []string{},
labels: []string{}, // No specific capabilities, matches jobs with empty requirements? No, empty labels matches nothing?
// Wait, previous logic was: if reqLabels is empty, return all.
// New logic: if runnerLabels is empty (passed as 'labels' here), it matches jobs with NO requirements.
// But for test purposes, let's assume we pass runner capabilities.
// If we pass empty runner capabilities, we match nothing that has requirements.
// Let's pass capabilities that cover the jobs.
mockResponse: ActionWorkflowJobsResponse{
TotalCount: 3,
Jobs: []ActionWorkflowJob{
{ID: 1, Status: "queued", Labels: []string{"linux", "x64"}},
{ID: 2, Status: "queued", Labels: []string{"windows"}},
{ID: 3, Status: "queued", Labels: []string{"macos"}},
{ID: 1, Status: "queued", Labels: []string{"linux"}},
},
},
expectedCount: 3,
expectedError: false,
expectedQueued: 0, // No runner capabilities provided -> no match
expectedError: false,
},
{
name: "global scope with specific labels",
scope: v1alpha1.RunnerGroupScopeGlobal,
labels: []string{"docker"},
labels: []string{"docker", "linux"},
mockResponse: ActionWorkflowJobsResponse{
TotalCount: 2,
Jobs: []ActionWorkflowJob{
{ID: 1, Status: "queued", Labels: []string{"docker", "linux"}},
{ID: 2, Status: "queued", Labels: []string{"linux"}},
{ID: 1, Status: "queued", Labels: []string{"docker", "linux"}}, // Match
{ID: 2, Status: "queued", Labels: []string{"linux"}}, // Match (subset)
},
},
expectedCount: 1,
expectedError: false,
expectedQueued: 2,
expectedError: false,
},
{
name: "user scope",
scope: v1alpha1.RunnerGroupScopeUser,
user: "testuser",
labels: []string{"linux"},
mockResponse: ActionWorkflowJobsResponse{
TotalCount: 1,
Jobs: []ActionWorkflowJob{
{ID: 1, Status: "queued", Labels: []string{"linux"}},
},
},
expectedQueued: 1,
expectedError: false,
},
}
@@ -90,6 +108,23 @@ func TestHTTPClient_GetQueuedRuns(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
// Create mock server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
// Handle User Repos call for User Scope
if tt.scope == v1alpha1.RunnerGroupScopeUser && strings.Contains(r.URL.Path, "/repos") && !strings.Contains(r.URL.Path, "/actions/jobs") {
repos := []Repository{
{
Name: "testrepo",
Owner: struct {
Login string `json:"login"`
}{Login: tt.user},
FullName: tt.user + "/testrepo",
},
}
_ = json.NewEncoder(w).Encode(repos)
return
}
// Verify correct endpoint is called
expectedPath := ""
switch tt.scope {
@@ -99,35 +134,37 @@ func TestHTTPClient_GetQueuedRuns(t *testing.T) {
expectedPath = "/api/v1/orgs/testorg/actions/jobs"
case v1alpha1.RunnerGroupScopeGlobal:
expectedPath = "/api/v1/admin/actions/jobs"
case v1alpha1.RunnerGroupScopeUser:
expectedPath = "/api/v1/repos/" + tt.user + "/testrepo/actions/jobs"
}
if !strings.HasPrefix(r.URL.Path, expectedPath) {
t.Errorf("Expected path to start with %s, got %s", expectedPath, r.URL.Path)
}
// Verify query parameters
if r.URL.Query().Get("status") != "queued" {
t.Errorf("Expected status=queued, got %s", r.URL.Query().Get("status"))
}
// Verify authorization header
authHeader := r.Header.Get("Authorization")
if !strings.HasPrefix(authHeader, "token ") {
t.Errorf("Expected Authorization header to start with 'token ', got %s", authHeader)
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(tt.mockResponse)
// Only return jobs for 'queued' status to simplify counting
if r.URL.Query().Get("status") == "queued" {
_ = json.NewEncoder(w).Encode(tt.mockResponse)
} else {
_ = json.NewEncoder(w).Encode(ActionWorkflowJobsResponse{TotalCount: 0, Jobs: []ActionWorkflowJob{}})
}
}))
defer server.Close()
client := NewHTTPClient()
count, err := client.GetQueuedRuns(
stats, err := client.GetRunnerStats(
context.Background(),
server.URL,
"test-token",
tt.scope,
tt.org,
tt.user,
tt.repo,
tt.labels,
)
@@ -138,8 +175,10 @@ func TestHTTPClient_GetQueuedRuns(t *testing.T) {
if !tt.expectedError && err != nil {
t.Errorf("Expected no error but got: %v", err)
}
if count != tt.expectedCount {
t.Errorf("Expected count %d, got %d", tt.expectedCount, count)
if stats != nil {
if len(stats.QueuedJobs) != tt.expectedQueued {
t.Errorf("Expected %d queued jobs, got %d", tt.expectedQueued, len(stats.QueuedJobs))
}
}
})
}
@@ -149,46 +188,46 @@ func TestJobMatchesLabels(t *testing.T) {
client := &HTTPClient{}
tests := []struct {
name string
jobLabels []string
requiredLabels []string
expected bool
name string
jobLabels []string
supportedLabels []string
expected bool
}{
{
name: "exact match",
jobLabels: []string{"linux", "x64"},
requiredLabels: []string{"linux", "x64"},
expected: true,
name: "exact match",
jobLabels: []string{"linux", "x64"},
supportedLabels: []string{"linux", "x64"},
expected: true,
},
{
name: "subset match",
jobLabels: []string{"linux", "x64", "docker"},
requiredLabels: []string{"linux", "x64"},
expected: true,
name: "subset match (runner has more)",
jobLabels: []string{"linux"},
supportedLabels: []string{"linux", "x64"},
expected: true,
},
{
name: "no match",
jobLabels: []string{"linux", "arm64"},
requiredLabels: []string{"linux", "x64"},
expected: false,
name: "schema match",
jobLabels: []string{"ubuntu-latest"},
supportedLabels: []string{"ubuntu-latest:docker://node:16"},
expected: true,
},
{
name: "empty required labels",
jobLabels: []string{"linux", "x64"},
requiredLabels: []string{},
expected: true,
name: "no match (missing req)",
jobLabels: []string{"linux", "arm64"},
supportedLabels: []string{"linux", "x64"},
expected: false,
},
{
name: "partial match",
jobLabels: []string{"linux"},
requiredLabels: []string{"linux", "x64"},
expected: false,
name: "empty required labels (matches anything)",
jobLabels: []string{},
supportedLabels: []string{"linux"},
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := client.jobMatchesLabels(tt.jobLabels, tt.requiredLabels)
result := client.jobMatchesLabels(tt.jobLabels, tt.supportedLabels)
if result != tt.expected {
t.Errorf("Expected %v, got %v", tt.expected, result)
}
@@ -207,42 +246,32 @@ func TestFilterQueuedJobs(t *testing.T) {
}
tests := []struct {
name string
requiredLabels []string
expectedCount int
name string
supportedLabels []string
expectedIDs []int64
}{
{
name: "filter by linux",
requiredLabels: []string{"linux"},
expectedCount: 3,
name: "runner supports linux, x64",
supportedLabels: []string{"linux", "x64"},
expectedIDs: []int64{1},
},
{
name: "filter by linux and x64",
requiredLabels: []string{"linux", "x64"},
expectedCount: 2,
name: "runner supports linux, x64, docker",
supportedLabels: []string{"linux", "x64", "docker"},
expectedIDs: []int64{1, 4},
},
{
name: "filter by docker",
requiredLabels: []string{"docker"},
expectedCount: 1,
},
{
name: "no labels - return all",
requiredLabels: []string{},
expectedCount: 4,
},
{
name: "no matches",
requiredLabels: []string{"macos"},
expectedCount: 0,
name: "runner supports everything",
supportedLabels: []string{"linux", "x64", "arm64", "windows", "docker"},
expectedIDs: []int64{1, 2, 3, 4},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
count := client.filterQueuedJobs(jobs, tt.requiredLabels)
if count != tt.expectedCount {
t.Errorf("Expected %d, got %d", tt.expectedCount, count)
matched := client.filterQueuedJobs(jobs, tt.supportedLabels)
if len(matched) != len(tt.expectedIDs) {
t.Errorf("Expected %d matched jobs, got %d", len(tt.expectedIDs), len(matched))
}
})
}