ham/hamview
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Checkpoint
Some checks failed
Test and build / Test and lint (push) Failing after 36s
Details
Test and build / Build collector (push) Failing after 43s
Details
Test and build / Build receiver (push) Failing after 42s
Details

Browse Source
This commit is contained in:
maze
2026-03-05 15:38:18 +01:00
parent 3106b2cf45
commit 13afa08e8a
108 changed files with 19509 additions and 729 deletions
Download Patch File Download Diff File Expand all files Collapse all files

591
collector.go
View File

@@ -1,15 +1,22 @@
package hamview
import (
"context"
"database/sql"
"encoding/hex"
"fmt"
"strings"
"time"
_ "github.com/cridenour/go-postgis" // PostGIS support
"github.com/lib/pq" // PostgreSQL support
_ "github.com/lib/pq" // PostgreSQL support
"xorm.io/builder"
"xorm.io/xorm"
"git.maze.io/go/ham/protocol"
"git.maze.io/go/ham/protocol/aprs"
"git.maze.io/go/ham/protocol/meshcore"
"git.maze.io/ham/hamview/schema"
)
type CollectorConfig struct {
@@ -22,58 +29,20 @@ type DatabaseConfig struct {
}
type Collector struct {
*sql.DB
radioByID map[string]*schema.Radio
meshCoreGroup map[byte][]*meshcore.Group
}
func NewCollector(config *CollectorConfig) (*Collector, error) {
d, err := sql.Open(config.Database.Type, config.Database.Conf)
if err != nil {
Logger.Debugf("collector: opening %q database", config.Database.Type)
schema.Logger = Logger
if err := schema.Open(config.Database.Type, config.Database.Conf); err != nil {
return nil, err
}
for _, query := range []string{
// radio.*
sqlCreateRadio,
sqlIndexRadioName,
sqlIndexRadioProtocol,
sqlGeometryRadioPosition,
// meshcore_packet.*
sqlCreateMeshCorePacket,
sqlIndexMeshCorePacketHash,
sqlIndexMeshCorePacketPayloadType,
// meshcore_node.*
sqlCreateMeshCoreNode,
sqlIndexMeshCoreNodeName,
sqlAlterMeshCoreNodePrefix,
sqlGeometryMeshCoreNodePosition,
// meshcore_node_position.*
sqlCreateMeshCoreNodePosition,
sqlGeometryMeshCoreNodePositionPosition,
sqlIndexMeshCoreNodePositionPosition,
} {
if _, err := d.Exec(query); err != nil {
var ignore bool
if err, ok := err.(*pq.Error); ok {
switch err.Code {
case "42701": // column "x" of relation "y" already exists (42701)
ignore = true
}
}
Logger.Debugf("collector: sql error %T: %v", err, err)
if !ignore {
return nil, fmt.Errorf("error in query %s: %v", query, err)
}
}
}
return &Collector{
DB: d,
meshCoreGroup: make(map[byte][]*meshcore.Group),
radioByID: make(map[string]*schema.Radio),
}, nil
}
@@ -92,6 +61,8 @@ func (c *Collector) Collect(broker Broker, topic string) error {
return err
}
ctx := context.Background()
loop:
for {
select {
@@ -99,7 +70,7 @@ loop:
if radio == nil {
break loop
}
c.processRadio(radio)
c.processRadio(ctx, radio)
case packet := <-packets:
if packet == nil {
@@ -107,9 +78,9 @@ loop:
}
switch packet.Protocol {
case protocol.APRS:
c.processAPRSPacket(packet)
c.processAPRSPacket(ctx, packet)
case protocol.MeshCore:
c.processMeshCorePacket(packet)
c.processMeshCorePacket(ctx, packet)
}
}
}
@@ -119,150 +90,182 @@ loop:
return nil
}
func (c *Collector) processRadio(radio *Radio) {
Logger.Tracef("collector: process %s radio %q online %t",
radio.Protocol,
radio.Name,
radio.IsOnline)
var latitude, longitude, altitude *float64
if radio.Position != nil {
latitude = &radio.Position.Latitude
longitude = &radio.Position.Longitude
altitude = &radio.Position.Altitude
func (c *Collector) getRadioByID(ctx context.Context, id string) (*schema.Radio, error) {
id = strings.TrimRight(id, "=")
if radio, ok := c.radioByID[id]; ok {
return radio, nil
}
var id int64
if err := c.QueryRow(`
INSERT INTO radio (
name,
is_online,
device,
manufacturer,
firmware_date,
firmware_version,
antenna,
modulation,
protocol,
latitude,
longitude,
altitude,
frequency,
rx_frequency,
tx_frequency,
bandwidth,
power,
gain,
lora_sf,
lora_cr,
extra
) VALUES (
$1,
$2,
NULLIF($3, ''), -- device
NULLIF($4, ''), -- manufacturer
$5,
NULLIF($6, ''), -- firmware_version
NULLIF($7, ''), -- antenna
NULLIF($8, ''), -- modulation
$9, -- protocol
NULLIF($10, 0.0), -- latitude
NULLIF($11, 0.0), -- longitude
$12, -- altitude
$13, -- frequency
NULLIF($14, 0.0), -- rx_frequency
NULLIF($15, 0.0), -- tx_frequency
$16, -- bandwidth
NULLIF($17, 0.0), -- power
NULLIF($18, 0.0), -- gain
NULLIF($19, 0), -- lora_sf
NULLIF($20, 0), -- lora_cr
$21
)
ON CONFLICT (name)
DO UPDATE
SET
is_online = $2,
device = NULLIF($3, ''),
manufacturer = NULLIF($4, ''),
firmware_date = $5,
firmware_version = NULLIF($6, ''),
antenna = NULLIF($7, ''),
modulation = NULLIF($8, ''),
protocol = $9,
latitude = NULLIF($10, 0.0),
longitude = NULLIF($11, 0.0),
altitude = $12,
frequency = $13,
rx_frequency = NULLIF($14, 0.0),
tx_frequency = NULLIF($15, 0.0),
bandwidth = $16,
power = NULLIF($17, 0),
gain = NULLIF($18, 0),
lora_sf = NULLIF($19, 0),
lora_cr = NULLIF($20, 0),
extra = $21
RETURNING id
`,
radio.Name,
radio.IsOnline,
radio.Device,
radio.Manufacturer,
radio.FirmwareDate,
radio.FirmwareVersion,
radio.Antenna,
radio.Modulation,
radio.Protocol,
latitude,
longitude,
altitude,
radio.Frequency,
radio.RXFrequency,
radio.TXFrequency,
radio.Bandwidth,
radio.Power,
radio.Gain,
radio.LoRaSF,
radio.LoRaCR,
nil,
).Scan(&id); err != nil {
Logger.Warnf("collector: error storing radio: %v", err)
radio, err := schema.GetRadioByEncodedID(ctx, id)
if err == nil {
c.radioByID[id] = radio
}
return radio, err
}
func (c *Collector) processRadio(ctx context.Context, received *Radio) {
Logger.Tracef("collector: process %s radio %q online %t",
received.Protocol,
received.Name,
received.IsOnline)
var (
now = time.Now()
engine = schema.Query(ctx).(*xorm.Session)
)
if err := engine.Begin(); err != nil {
Logger.Warnf("collector: can't start session: %v", err)
return
}
radio := new(schema.Radio)
has, err := engine.Where(builder.Eq{
"name": received.Name,
"protocol": received.Protocol,
}).Get(radio)
if err != nil {
Logger.Warnf("collector: can't query radio: %v", err)
return
}
if has {
radio.IsOnline = received.IsOnline
radio.UpdatedAt = now
if _, err = engine.Cols("is_online", "updated_at").Update(radio); err != nil {
_ = engine.Rollback()
Logger.Warnf("collector: can't update radio: %v", err)
return
}
} else {
radio = &schema.Radio{
Name: received.Name,
IsOnline: received.IsOnline,
Manufacturer: received.Manufacturer,
Device: schema.NULLString(received.Device),
FirmwareVersion: schema.NULLString(received.FirmwareVersion),
FirmwareDate: schema.NULLTime(received.FirmwareDate),
Antenna: schema.NULLString(received.Antenna),
Modulation: received.Modulation,
Protocol: received.Protocol,
Frequency: received.Frequency,
Bandwidth: received.Bandwidth,
Power: schema.NULLFloat64(received.Power),
Gain: schema.NULLFloat64(received.Gain),
CreatedAt: now,
UpdatedAt: now,
}
if received.Position != nil {
radio.Latitude = &received.Position.Latitude
radio.Longitude = &received.Position.Longitude
radio.Altitude = &received.Position.Altitude
}
if received.LoRaCR != 0 && received.LoRaSF != 0 {
radio.LoRaSF = &received.LoRaSF
radio.LoRaCR = &received.LoRaCR
}
if _, err = engine.Insert(radio); err != nil {
Logger.Warnf("collector: can't insert radio %#+v: %v", radio, err)
return
}
}
if err = engine.Commit(); err != nil {
Logger.Errorf("collector: can't commit radio session: %v", err)
}
}
func (c *Collector) processAPRSPacket(packet *protocol.Packet) {
decoded, err := aprs.ParsePacket(string(packet.Raw))
func (c *Collector) processAPRSPacket(ctx context.Context, received *Packet) {
radio, err := c.getRadioByID(ctx, received.RadioID)
if err != nil {
Logger.Warnf("collector: invalid %s packet: %v", packet.Protocol, err)
Logger.Warnf("collector: process %s packet: can't find radio %q: %v", received.Protocol, received.RadioID, err)
return
}
decoded, err := aprs.Parse(string(received.Raw))
if err != nil {
Logger.Warnf("collector: invalid %s packet: %v", received.Protocol, err)
return
}
Logger.Tracef("collector: process %s packet (%d bytes)",
packet.Protocol,
len(packet.Raw))
received.Protocol,
len(received.Raw))
var id int64
if err := c.QueryRow(`
INSERT INTO aprs_packet (
src_address,
dst_address,
comment
) VALUES ($1, $2, $3)
RETURNING id;
`,
decoded.Src.String(),
decoded.Dst.String(),
decoded.Comment,
).Scan(&id); err != nil {
Logger.Warnf("collector: error storing packet: %v", err)
engine := schema.Query(ctx)
station := new(schema.APRSStation)
has, err := engine.Where(builder.Eq{"call": strings.ToUpper(decoded.Source.String())}).Get(station)
if err != nil {
Logger.Warnf("collector: can't query APRS station: %v", err)
return
} else if has {
cols := []string{"last_heard_at"}
station.LastHeardAt = received.Time
if decoded.Latitude != 0 {
station.LastLatitude = sql.NullFloat64{Float64: decoded.Latitude, Valid: true}
station.LastLongitude = sql.NullFloat64{Float64: decoded.Longitude, Valid: true}
cols = append(cols, "last_latitude", "last_longitude")
}
if _, err = engine.ID(station.ID).Cols(cols...).Update(station); err != nil {
Logger.Warnf("collector: can't update APRS station: %v", err)
return
}
} else {
station = &schema.APRSStation{
Call: strings.ToUpper(decoded.Source.String()),
Symbol: decoded.Symbol,
FirstHeardAt: received.Time,
LastHeardAt: received.Time,
}
if decoded.Latitude != 0 {
station.LastLatitude = sql.NullFloat64{
Float64: decoded.Latitude,
Valid: decoded.Latitude != 0,
}
station.LastLongitude = sql.NullFloat64{
Float64: decoded.Longitude,
Valid: decoded.Longitude != 0,
}
}
if station.ID, err = engine.Insert(station); err != nil {
Logger.Warnf("collector: can't insert APRS station: %v", err)
return
}
}
packet := &schema.APRSPacket{
RadioID: radio.ID,
StationID: station.ID,
Source: station.Call,
Destination: decoded.Destination.String(),
Path: decoded.Path.String(),
Comment: decoded.Comment,
Symbol: string(decoded.Symbol[:]),
Raw: string(received.Raw),
}
if decoded.Latitude != 0 {
packet.Latitude = sql.NullFloat64{
Float64: decoded.Latitude,
Valid: decoded.Latitude != 0,
}
packet.Longitude = sql.NullFloat64{
Float64: decoded.Longitude,
Valid: decoded.Longitude != 0,
}
}
if _, err = engine.Insert(packet); err != nil {
Logger.Warnf("collector: can't insert APRS packet: %v", err)
}
}
func (c *Collector) processMeshCorePacket(packet *protocol.Packet) {
func (c *Collector) processMeshCorePacket(ctx context.Context, packet *Packet) {
radio, err := c.getRadioByID(ctx, packet.RadioID)
if err != nil {
Logger.Warnf("collector: process %s packet: can't find radio %q: %v", packet.Protocol, packet.RadioID, err)
return
}
var parsed meshcore.Packet
if err := parsed.UnmarshalBytes(packet.Raw); err != nil {
if err = parsed.UnmarshalBytes(packet.Raw); err != nil {
Logger.Warnf("collector: invalid %s packet: %v", packet.Protocol, err)
return
}
@@ -276,125 +279,137 @@ func (c *Collector) processMeshCorePacket(packet *protocol.Packet) {
parsed.Path = nil // store NULL
}
var id int64
if err := c.QueryRow(`
INSERT INTO meshcore_packet (
snr,
rssi,
hash,
route_type,
payload_type,
path,
payload,
raw,
received_at
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
RETURNING id;`,
packet.SNR,
packet.RSSI,
parsed.Hash(),
parsed.RouteType,
parsed.PayloadType,
parsed.Path,
parsed.Payload,
packet.Raw,
packet.Time,
).Scan(&id); err != nil {
var channelHash string
switch parsed.PayloadType {
case meshcore.TypeGroupText, meshcore.TypeGroupData:
if len(parsed.Payload) > 0 {
channelHash = fmt.Sprintf("%02x", parsed.Payload[0])
}
}
var (
save = &schema.MeshCorePacket{
RadioID: radio.ID,
SNR: packet.SNR,
RSSI: packet.RSSI,
RouteType: uint8(parsed.RouteType),
PayloadType: uint8(parsed.PayloadType),
Version: parsed.Version,
Hash: hex.EncodeToString(parsed.Hash()),
Path: parsed.Path,
Payload: parsed.Payload,
ChannelHash: channelHash,
Raw: packet.Raw,
ReceivedAt: packet.Time,
}
)
if _, err = schema.Query(ctx).Insert(save); err != nil {
Logger.Warnf("collector: error storing packet: %v", err)
return
}
switch parsed.PayloadType {
case meshcore.TypeAdvert:
payload, err := parsed.Decode()
if err != nil {
Logger.Warnf("collector: error decoding packet: %v", err)
return
}
var (
advert = payload.(*meshcore.Advert)
nodeID int64
latitude *float64
longitude *float64
)
if advert.Position != nil {
latitude = &advert.Position.Latitude
longitude = &advert.Position.Longitude
}
if err = c.QueryRow(`
INSERT INTO meshcore_node (
node_type,
public_key,
name,
local_time,
first_heard,
last_heard,
last_latitude,
last_longitude,
last_advert_id
) VALUES (
$1,
$2,
$3,
$4,
$5,
$6,
$7,
$8,
$9
)
ON CONFLICT (public_key)
DO UPDATE
SET
name = $3,
local_time = $4,
last_heard = $6,
last_latitude = $7,
last_longitude = $8,
last_advert_id = $9
RETURNING id
`,
advert.Type,
advert.PublicKey.Bytes(),
advert.Name,
advert.Time,
packet.Time,
packet.Time,
latitude,
longitude,
id,
).Scan(&nodeID); err != nil {
Logger.Warnf("collector: error storing node: %v", err)
return
}
if advert.Position != nil {
if _, err = c.Exec(`
INSERT INTO meshcore_node_position (
node_id,
heard_at,
latitude,
longitude,
position
) VALUES (
$1,
$2,
$3,
$4,
ST_SetSRID(ST_MakePoint($5, $6), 4326)
);
`,
nodeID,
packet.Time,
advert.Position.Latitude,
advert.Position.Longitude,
advert.Position.Latitude,
advert.Position.Longitude,
); err != nil {
Logger.Warnf("collector: error storing node position: %v", err)
return
}
}
c.processMeshCoreAdvert(ctx, save, &parsed)
}
}
func (c *Collector) processMeshCoreAdvert(ctx context.Context, packet *schema.MeshCorePacket, parsed *meshcore.Packet) {
payload, err := parsed.Decode()
if err != nil {
Logger.Warnf("collector: error decoding packet: %v", err)
return
}
advert, ok := payload.(*meshcore.Advert)
if !ok {
Logger.Warnf("collector: expected Advert, got %T!?", payload)
return
}
node := &schema.MeshCoreNode{
PacketHash: packet.Hash,
Name: advert.Name,
Type: uint8(advert.Type),
Prefix: fmt.Sprintf("%02x", advert.PublicKey.Bytes()[0]),
PublicKey: hex.EncodeToString(advert.PublicKey.Bytes()),
FirstHeardAt: packet.ReceivedAt,
LastHeardAt: packet.ReceivedAt,
}
if advert.Position != nil {
node.LastLatitude = &advert.Position.Latitude
node.LastLongitude = &advert.Position.Longitude
}
var (
engine = schema.Query(ctx)
existing = new(schema.MeshCoreNode)
)
if err = engine.Begin(); err != nil {
Logger.Warnf("collector: can't start session: %v", err)
return
}
var has bool
if has, err = engine.Where(builder.Eq{"`public_key`": node.PublicKey}).Get(existing); err != nil {
_ = engine.Rollback()
Logger.Warnf("collector: can't query session: %v", err)
return
}
if has {
cols := []string{"last_heard_at"}
existing.LastHeardAt = packet.ReceivedAt
if advert.Position != nil {
existing.LastLatitude = node.LastLatitude
existing.LastLongitude = node.LastLongitude
cols = append(cols, "last_latitude", "last_longitude")
}
existing.Name = node.Name
_, err = engine.ID(existing.ID).Cols(cols...).Update(existing)
} else {
_, err = engine.Insert(node)
}
if err != nil {
_ = engine.Rollback()
Logger.Warnf("collector: can't save (update: %t): %v", has, err)
return
}
if err = engine.Commit(); err != nil {
Logger.Warnf("collector: can't commit session: %v", err)
return
}
/*
if advert.Position != nil {
if _, err = c.DB.Exec(`
INSERT INTO meshcore_node_position (
node_id,
heard_at,
latitude,
longitude,
position
) VALUES (
$1,
$2,
$3,
$4,
ST_SetSRID(ST_MakePoint($5, $6), 4326)
);
`,
nodeID,
packet.Time,
advert.Position.Latitude,
advert.Position.Longitude,
advert.Position.Latitude,
advert.Position.Longitude,
); err != nil {
Logger.Warnf("collector: error storing node position: %v", err)
return
}
}
}
*/
}