Python gRPC client does not connect to Golang gRPC server in a different pod

General Discussions
1

I have a Python client running in the POD-A pod trying to connect to a Golang server running in the POD-B pod and I am getting this error:

"<_InactiveRpcError of RPC that terminated with:\n\tstatus = StatusCode.UNAVAILABLE\n\tdetails = \"DNS resolution failed for service: pod-b\"\n\tdebug_error_string = \"{\"created\":\"@1649433162.980011551\",\"description\":\"Resolver transient failure\",\"file\":\"src/core/ext/filters/client_channel/client_channel.cc\",\"file_line\":1357,\"referenced_errors\":[{\"created\":\"@1649433162.979997474\",\"description\":\"DNS resolution failed for service: pod-b\",\"file\":\"src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc\",\"file_line\":359,\"grpc_status\":14,\"referenced_errors\":[{\"created\":\"@1649433162.979938651\",\"description\":\"C-ares status is not ARES_SUCCESS qtype=A name=pod-b is_balancer=0: Timeout while contacting DNS servers\",\"file\":\"src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc\",\"file_line\":724}]}]}\"\n>"

The Python client code is:

        channel = grpc.insecure_channel("pod-b")
        stub = campaign_service_pb2_grpc.CampaignServiceStub(channel)
        request = campaign_service_pb2.CreateCampaignRequest(amount=12345)
        response = stub.CreateCampaign(request)
        return response.id

The Golang server code is:

// Server starts a new gRPC Server
func New(conf config.Config, services *services.Services, logger config.Logger) (*Server, error) {
    flag.Parse()
    conn, err := net.Listen("tcp", fmt.Sprintf(conf.GRPC.Host+":%d", conf.GRPC.Port)) // host:127.0.0.1  port:50051
    if err != nil {
        return nil, errors.Wrap(err, "failed to listen")
    }
    server := Server{
        conn: 	conn,
        logger:   logger,
        services: services,
    }
    s := grpc.NewServer()
    pb.RegisterCampaignServiceServer(s, &server)
    server.s = s
    return &server, nil
}

docker-compose.yaml of the server:

version: "3"
 
services:
  pod-b:
    ports:
      - "7007:8080"
      - "50051:50051"
    build:
      context: .
      args:
        - DEPLOY_KEY=${DEPLOY_KEY}
    depends_on:
      - crdb
    environment:
      - BUDGET_MANAGER_COCKROACHDB_HOST=${BUDGET_MANAGER_COCKROACHDB_HOST}
      - BUDGET_MANAGER_COCKROACHDB_PORT=${BUDGET_MANAGER_COCKROACHDB_PORT}
      - BUDGET_MANAGER_COCKROACHDB_USER=${BUDGET_MANAGER_COCKROACHDB_USER}
      - BUDGET_MANAGER_COCKROACHDB_PASSWORD=${BUDGET_MANAGER_COCKROACHDB_PASSWORD}
      - BUDGET_MANAGER_COCKROACHDB_DB=${BUDGET_MANAGER_COCKROACHDB_DB}
      - BUDGET_MANAGER_COCKROACHDB_MIGRATE=${BUDGET_MANAGER_COCKROACHDB_MIGRATE}
      - BUDGET_MANAGER_COCKROACHDB_SSL=${BUDGET_MANAGER_COCKROACHDB_SSL}
      - BUDGET_MANAGER_COCKROACHDB_KEY=${BUDGET_MANAGER_COCKROACHDB_KEY}
      - BUDGET_MANAGER_COCKROACHDB_CERT=${BUDGET_MANAGER_COCKROACHDB_CERT}
 
  crdb:
    image: cockroachdb/cockroach:v21.2.5
    ports:
      - "26257:26257"
      - "8081:8080"
    command: start-single-node --insecure

service.yaml of the server:

apiVersion: v1
kind: Service
metadata:
 name: "{{ .Values.name }}"
 labels:
   app: "{{ .Values.name }}"
   monitor: "true"
spec:
 type: NodePort
 ports:
   - port: {{ .Values.servicePort }}
     name: http
     targetPort: {{ .Values.port }}
   - port: {{ .Values.grpc.servicePort }}
     name: grpc
     targetPort: {{ .Values.grpc.port }}
 selector:
   app: "{{ .Values.name }}"

values.yaml of the server:

deployment:
  image: pod-b
  tag: 0.1.0

name: pod-b

replicas: 2

resources:
  requests:
    memory: 50Mi
    cpu: 50m
  limits:
    memory: 300Mi
    cpu: 150m

http:
  port: 8080
  servicePort: 80

grpc:
  port: 50051
  servicePort: 50051

secret:
  create: false
  monitoring: "apps-metrics"

env:
  config:
    environment: staging
    port: 8080

cockroachdb:
  host: something
  port: 26257
  user: something
  db: something
  migrate: true
  ssl: true
  key: something
  cert: something

Does anyone know what is going on?